mike/linuxsetup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit [141.14.140.180,mike]

Browse Source
This commit is contained in:
mike
2026-08-10 16:12:09 +02:00
commit c534ea1c79
66 changed files with 9244 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
files/mbackup
Executable
+573
View File
@@ -0,0 +1,573 @@
#!/usr/bin/perl
############################################################################################# mbackup (mwx'2023)
$VERSION='3.2.0';
use Getopt::Std;getopts('vVDWMYhsfc:');$opt_v=1 if ($opt_V);
use Net::Ping;
use Fcntl ':flock';
chomp($HOST=`hostname`);
&help if($opt_h);
$NOW=time();
my(@z)=localtime($NOW);$z[4]+=1;$z[5]-=100;$z[6]+=1;
$z=sprintf("%02d.%02d-%02d:%02d",$z[3],$z[4],$z[5],$z[2],$z[1]);
$MAX{'daily'}=7; # default values
$MAX{'weekly'}=4;
$MAX{'monthly'}=12;
$MAX{'yearly'}=3;
$FLAT=0;
$RSYNCOPTS='--no-o --no-g --chmod=ugo=rwX';
$RSYNCRETRIES=3;
$CONFIG='';$REQUESTCONFIG=$ARGV[0]; # load configuration
for ($opt_c,"/db/etc/mbackup.conf",$ENV{'HOME'}."/etc/mbackup.conf",$ENV{'HOME'}."/.mbackup.conf") {
if (-f) {
$confile=$_;
last;
}
}
require $confile;
for (keys(%GROUPS)) { # build group list
if ($_ eq $REQUESTCONFIG) {
push(@GROUP,@{$GROUPS{$_}});
$CONFIG=$REQUESTCONFIG;
}
}
if ($CONFIG=~/^\s*$/) { # check for valid config
&log("ERROR: config '$REQUESTCONFIG' not found",1);
exit;
}
$REMOTE=0;$REMOTE=1 if ($TARGETDIR!~/^\//);
if ($opt_s) { # show configuration
if (@GROUP) {
print "\n";
print "mode: group\n\n";
print "group sleep: $GROUPSLEEP\n\n" if ($GROUPSLEEP>0);
for $job (@GROUP) {
print "/db/bin/mbackup -c $confile $job\n";
}
print "\n";
} else {
print "\n";
if ($FLAT==1) {
print "mode: flat\n";
} else {
print "mode: $MAX{'daily'} daily, $MAX{'weekly'} weekly, ";
print "$MAX{'monthly'} monthly, $MAX{'yearly'} yearly\n";
}
if ($#EXCLUDE>-1) {
print "\n";
$tmp="";$tmp.="$_, " for (@EXCLUDE);$tmp=~s/\,\s+$//;
print "exclude: $tmp\n";
}
print "\n";
$l=0;
for (keys(%SRC)) {
$l=length($_) if length($_)>$l;
}
printf "%-${l}s -> %s\/%s\n",$_,$TARGETDIR,$SRC{$_} for (keys(%SRC));
print "\n";
}
exit;
}
if (@GROUP) { # run backup group
for $job (@GROUP) {
$cmd="/db/bin/mbackup -c $confile $job";
&log("{$CONFIG} $cmd",1);
system($cmd);
if ($GROUPSLEEP>0 && $job ne $GROUP[-1]) {
&log("group loop sleep ($GROUPSLEEP seconds)",1);
sleep($GROUPSLEEP);
}
}
exit;
}
if ($MY_LOG) { # mysql loging
use DBI;
$dbh=DBI->connect("DBI:mysql:$MY_DB:$MY_HOST:",$MY_USER,$MY_PASSWD,{RaiseError=>1,PrintError=>1});
}
$mode=''; # set mode (daily,weekly,monthly,yearly)
my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($NOW);
$mon++;$year+=1900;
$z=sprintf("%02d\.%02d\.%04d %02d\:%02d",$mday,$mon,$year,$hour,$min);
$mode='daily' if ( $MAX{'daily'}>0);
$mode='weekly' if ( $wday==0 && $MAX{'weekly'}>0);
$mode='monthly' if ( $mday==1 && $MAX{'monthly'}>0);
$mode='yearly' if ( $yday==0 && $MAX{'yearly'}>0);
$mode='daily' if ($opt_D); # set mode manual
$mode='weekly' if ($opt_W);
$mode='monthly' if ($opt_M);
$mode='yearly' if ($opt_Y);
if (defined($PING)) { # check if host is reachable
if (!Net::Ping->new("icmp", 15)->ping($PING)) {
&log("ERROR: host $PING not reachable",1);exit;
}
}
if (!-d "$TARGETDIR" && $REMOTE==0) { # check target dir
if ($opt_f) {
system("mkdir $TARGETDIR");
if (!-d "$TARGETDIR") {
&log("ERROR: target dir creation failed (mkdir $TARGETDIR)",1);
exit;
}
} else {
&log("ERROR: can not access target dir (mkdir $TARGETDIR)",1);
exit;
}
}
for $srcdir (keys(%SRC)) { # check source dir
if ($srcdir=~/^\// && !-d $srcdir) {
&log("ERROR: mising source dir '$srcdir'",1);
exit;
}
}
undef(%keycount); # check for duplicate source keys
for $srcdir (keys(%SRC)) {
$keycount{$SRC{$srcdir}}++;
if ($keycount{$SRC{$srcdir}}>1) {
&log("ERROR: duplicate source key '$SRC{$srcdir}'",1);
exit;
}
}
if ($FLAT==1 || $REMOTE==1) { # flat mode
$toterr=0;$totfiles=0;$tottfi=0;$totbytes=0;$tottby=0;$totrt=0;
for $srcdir (keys(%SRC)) {
$subname=$SRC{$srcdir};
$cmd="/usr/bin/rsync -av $RSYNCOPTS --stats --delete " .
&exf() . " --numeric-ids $srcdir/. $TARGETDIR/$subname";
($err,$fi,$tfi,$by,$tby,$rt,$try)=&rsync($cmd);
&log("[$CONFIG] $srcdir,$TARGETDIR ($err,$tfi,$fi,$tby,$by,$rt)",1);
if ($MY_LOG) { # mysql logging
$sql="INSERT INTO mbackup VALUES(0,now(),'$HOST','$VERSION','$CONFIG',".
"'$srcdir','$TARGETDIR',$fi,$tfi,$by,$tby,$rt,$err,$try)";
$dbh->do($sql);
}
if ($HTTP_LOG!~/^\s*$/) { # http logging
$tmp= "MBACKUP,$HOST,$VERSION,$CONFIG,$srcdir,$TARGETDIR,$fi,$tfi,$by,$tby,$rt,$err,$try";
use LWP::UserAgent;
my $ua = new LWP::UserAgent();
$response = $ua->post("http://$HTTP_LOG", Content => $tmp);
}
$toterr+=$err;$totfi+=$fi;$tottfi+=$tfi;$totby+=$by;$tottby+=$tby;$totrt+=$rt;
&log("rsync error $err in $srcdir/. -> $TARGETDIR/$subname") if ($err);
}
if ($toterr==0) {
&log("[$CONFIG] mbackup successful finished ($tottfi,$totfi,$tottby,$totby,$totrt)",1)
}
exit;
}
if ($mode eq '') { # check if mode is defiend otherwise exit
&log("ERROR: undefined mode",1);
exit;
}
$yweekly=0;$ymonthly=0;$yyearly=0; # search for missing snapshot
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
my(@stat)=stat("$TARGETDIR/$file");
if ($file=~/^weekly\.\d+$/) {
$yweekly=$stat[9] if ($stat[9]>$yweekly);
}
if ($file=~/^monthly\.\d+$/) {
$ymonthly=$stat[9] if ($stat[9]>$ymonthly);
}
if ($file=~/^yearly\.\d+$/) {
$yyearly=$stat[9] if ($stat[9]>$yyearly);
}
}
closedir(DIR);
if ($NOW-$yweekly>86400*10 && $yweekly!=0 && $MAX{'weekly'}>0) {
&log("WARNING: no weekly snapshot for more than 10 days",1);
}
if ($NOW-$ymonthly>86400*40 && $ymonthly!=0 && $MAX{'monthly'}>0) {
&log("WARNING: no monthly snapshot for more than 40 days",1);
}
if ($NOW-$yyearly>86400*400 && $yyearly!=0 && $MAX{'yearly'}>0) {
&log("WARNING: no yearly snapshot for more than 400 days",1);
}
undef(%SNAPLIST); # read snapshot list for current mode
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
my(@stat)=stat("$TARGETDIR/$file");
$SNAPLIST{$1}=$stat[9] if ($file=~/^$mode\.(\d+)/);
}
closedir(DIR);
for $n (sort {$b<=>$a} keys(%SNAPLIST)) { # snapshot cycling
$m=$n+1;
if (-d "$TARGETDIR/$mode.$n" && !-e "$TARGETDIR/$mode.$m") {
if ($m>=$MAX{$mode}) {
&cmd("rm -rf $TARGETDIR/$mode.$n");
} else {
&cmd("mv $TARGETDIR/$mode.$n $TARGETDIR/$mode.$m");
}
} else {
$tmp="";
$tmp="($TARGETDIR/$mode.$n not found)" if (!-d "$TARGETDIR/$mode.$n");
$tmp="($TARGETDIR/$mode.$m exists)" if (-e "$TARGETDIR/$mode.$m");
&log("ERROR: error while snapshot cycling $tmp",1);exit;
}
}
$youngest='';$y=0; # search for youngest snapshot
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
my(@stat)=stat("$TARGETDIR/$file");
if ($file=~/^((daily|weekly|monthly|yearly)\.\d+)/) {
if ($stat[9]>$y) {
$y=$stat[9];
$youngest=$file;
}
}
}
closedir(DIR);
&cmd("mkdir -m 0755 -p $TARGETDIR/$mode.tmp/"); # create new $mode.0
&cmd("touch $TARGETDIR/$mode.tmp/");
if (-d "$TARGETDIR/$mode.tmp") { # build new snapshot
$toterr=0;$totfi=0;$tottfi=0;$totbytes=0;$tottby=0;
for $srcdir (keys(%SRC)) {
$subname=$SRC{$srcdir};
$linkdest="";
if ($youngest!~/^\s*$/) {
if (-d "$TARGETDIR/$youngest" && -d "$TARGETDIR/$youngest/$subname") {
$linkdest="--link-dest=$TARGETDIR/$youngest/$subname"
}
}
$cmd="/usr/bin/rsync -av $RSYNCOPTS --no-o --no-g --chmod=ugo=rwX --stats --delete " .
&exf() . " --numeric-ids $linkdest $srcdir/. $TARGETDIR/$mode.tmp/$subname";
($err,$fi,$tfi,$by,$tby,$rt,$try)=&rsync($cmd);
&log("[$CONFIG] $srcdir,$TARGETDIR ($err,$tfi,$fi,$tby,$by,$rt)",1);
if ($MY_LOG) { # mysql logging
$sql="INSERT INTO mbackup VALUES(0,now(),'$HOST','$VERSION','$CONFIG',".
"'$srcdir','$TARGETDIR',$fi,$tfi,$by,$tby,$rt,$err,$try)";
$dbh->do($sql);
}
if ($HTTP_LOG!~/^\s*$/) { # http logging
$tmp= "MBACKUP,$HOST,$VERSION,$CONFIG,$srcdir,$TARGETDIR,$fi,$tfi,$by,$tby,$rt,$err,$try";
use LWP::UserAgent;
my $ua = new LWP::UserAgent();
$response = $ua->post("http://$HTTP_LOG", Content => $tmp);
}
$toterr+=$err;$totfi+=$fi;$tottfi+=$tfi;$totby+=$by;$tottby+=$tby;$totrt+=$rt;
}
if ($toterr==0) {
&log("[$CONFIG] mbackup successful finished ($tottfi,$totfi,$tby,$by,$totrt)",1);
&cmd("mv $TARGETDIR/$mode.tmp $TARGETDIR/$mode.0");
utime $NOW,$NOW,"$TARGETDIR/$mode.0";
}
} else {
&log("ERROR: can not access $mode.tmp",1);
}
######################################################################################################## SUPPORT
sub cmd() { # run command
my($cmd)=@_;
&log($cmd);
system($cmd);
}
sub log() { # write logs
my($str,$forceprint)=@_;
my(@z)=localtime(time());$z[4]+=1;$z[5]-=100;$z[6]+=1;
$ts=sprintf("%02d\.%02d\.%02d %02d\:%02d",$z[3],$z[4],$z[5],$z[2],$z[1]);
if ($LOGFILE!~/^\s+$/) {
open(LOG,">> $LOGFILE");
flock(LOG, LOCK_EX);
print LOG "$ts $$/$VERSION/$CONFIG $str\n";
close(LOG);
}
print "$str\n" if ($opt_v || $forceprint);
}
sub rsync() { # run rsync
my($rsynccmd)=@_;
my($files)=0;my($tfi)=0;my($bytes)=0;my($tby)=0;my($error)=0;my($try)=0;
$runtime=time();
$retries=0;
for $try (1..$RSYNCRETRIES) {
$rsynclog="";
open(RSYNC,"$rsynccmd |");while(<RSYNC>) {
print if ($opt_V);
if (/Number of files:\s*([\d\,]+)/) {;$files=$1;$files=~s/,//g;}
if (/Number of regular files transferred:\s*([\d\,]+)/) {;$tfi=$1;$tfi=~s/,//g;}
if (/Total file size:\s*([\d\,]*)\s*bytes/) {;$bytes=$1;$bytes=~s/,//g;}
if (/Total transferred file size:\s*([\d\,]*)\s*bytes/) {;$tby=$1;$tby=~s/,//g;}
}
close(RSYNC);
$error=($? >> 8);
$runtime=time()-$runtime;
if ($error==0) {
&log("$rsynccmd (successful)");
} else {
&log("$rsynccmd (error $error in try $try)")
}
last if ($error==0);
$retries++;
}
return ($error,$files,$tfi,$bytes,$tby,$runtime,$retries);
}
sub exf() { # exclude file
$ex='';
if ($#EXCLUDE>-1) {
$exfile="/tmp/mbackup.exclude.$CONFIG";
open(OUT,"> $exfile");
print OUT "$_\n" for (@EXCLUDE);
close(OUT);
$ex="--exclude-from $exfile --delete-excluded";
}
return $ex;
}
sub buildsrc() { # build source list (used in config)
my($base,@paths)=@_;
for $path (@paths) {
if ($path=~/\/([^\/]+)$/) {
$SRC{"$base$path"}=$1;
}
}
}
sub sc() { # build simple configuration (used in config)
my($uhd,$dirs,$exclude,$grp,$opt)=@_;
$mode=0;
if ($uhd=~/^([^\.]+)\.(.*)\:\:(.*)$/) {;$user=$3;$config=$1;$domain=$2;$mode=1;}
if ($uhd=~/^(\w+)\@([^\.]+)\.(.*)$/) {;$user=$1;$config=$2;$domain=$3;$mode=2;}
if ($mode>0) {
%opt=%$opt;
$config=$opt{name} if ($opt{name});
@grp=@$grp;
for $g (@grp) {
push(@{$GROUPS{$g}},$config)
}
if ($REQUESTCONFIG eq $config) {
$RSYNCOPTS=$opt{rsync} if ($opt{rsync});
$PING=$opt{ping} if ($opt{ping});
$TARGETBASE=$opt{target} if ($opt{target});
if ($opt{versions}=~/(\d+):(\d+):(\d+):(\d+)/) {
$MAX{'daily'}=$1;
$MAX{'weekly'}=$2;
$MAX{'monthly'}=$3;
$MAX{'yearly'}=$4;
$FLAT=0;
} else {
$FLAT=1;
}
@DIRS=@$dirs;
@EXCLUDE=@$exclude;
$TARGETDIR="$TARGETBASE/$config";
for $path (@DIRS) {
$tpath=$path;$tpath=~s/^\s*\///;$tpath=~s/\//\./g;
$SRC{"$uhd$path"}=$tpath if ($mode==1);
$SRC{"$uhd:$path"}=$tpath if ($mode==2);
}
$CONFIG=$REQUESTCONFIG;
}
}
}
sub help() { # help
print "mbackup v$VERSION\n";
print <<'ENDOFHELP';
USAGE:
------
mbackup [-v] [-V] [-DWMY] <backup job name>
mbackup -h
-D force daily mode
-W force weekly mode
-M force monthly mode
-Y force yearly mode
-v verbose output
-V very verbose output
-f force creation of target directory
-h this help
CONFIG FILE (/db/etc/mbackup.conf) GLOBAL OPTIONS:
--------------------------------------------------
$LOGFILE = "/var/log/mbackup.log"; # logfile location
$TARGETBASE = "/data/backup"; # targetbase for simple config
$HTTP_LOG = 'log.mxwx.org'; # http logging server
$MY_LOG = 1; # enable/disable mysql logging (deprecated)
$MY_HOST = 'localhost';
$MY_USER = '';
$MY_PASSWD = '';
$MY_DB = '';
SIMPLE BACKUP JOB CONFIGURATION (/db/etc/mbackup.conf):
-------------------------------------------------------
&sc(target,[paths],[exclude],[groups],{ option=>value, .. }
example:
&sc('root@host.domain',['/etc','/var','/home'],['/lib/lxcfs'],['all']);
&sc('host.domain::root',['/etc','/var','/home'],[],['all'], { rsync=>"-e 'ssh -p 2234'" });
&sc('root@host.domain',['/etc','/var','/home'],[],[] { versions=>'7:4:12:3' });
options:
name => 'name' # set configuration name
rsync => 'rsync options' # set additional rsync options
ping => 'ip address' # apply ping test before backup
version => 'daily:weekly:monthly:yearly' # define backup versioning
target => '/path/to/mbackup' # define backup destination
BACKUP JOB CONFIGURATION (/db/etc/mbackup.conf):
------------------------------------------------
if ($REQUESTCONFIG eq 'NAME') { # basic backup job config
<OPTIONS>
$CONFIG=$REQUESTCONFIG;
}
if ($REQUESTCONFIG eq 'NAME') { # backup group definition
@GROUP=('NAME1','NAME2',...);
$CONFIG=$REQUESTCONFIG;
}
push(@{$GROUPS{'all'}},'NAME'); # add config to group
BACKUP JOB OPTIONS:
-------------------
# backup target definition (use only once)
$TARGETDIR='/backup/NAME' # local traget directory
$TARGETDIR='10.0.1.1::root/backup/NAME' # rsync protocol target
$TARGETDIR='root@10.0.1.1:/backup/NAME' # ssh traget directory
# backup source definition (multiple allowd, different keys required)
$SRC{'/etc'} = 'etc1'; # local source
$SRC{'10.0.1.1::root/etc'} = 'etc2'; # remote source (rsync protocol)
$SRC{'root@10.0.1.1:/etc'} = 'etc3'; # remote source (ssh)
# define eculsions (absolut path definitions from source path)
@EXCLUDE=('/path1','/path2',...);
# define number of backups to keep, at least one value must be greater than 0
$MAX{'daily'} = 7; # keep n daily backup (default=7)
$MAX{'weekly'} = 4; # keep n weekly backup (default=4)
$MAX{'monthly'} = 12; # keep n monthly backup (default=12)
$MAX{'yearly'} = 3; # keep n yearly backup (default=3)
$FLAT=1; # only keep one backup (default=0)
$PING='10.0.1.1'; # perform ping test before backup run
$RSYNCOPTS="-e 'ssh -p 2234'"; # add extra options for rsync
$RSYNCRETRIES=3; # number of retries if rsync fails
MYSQL LOGGING TABLE:
--------------------
CREATE TABLE `mbackup` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`ts` timestamp NULL DEFAULT NULL,
`host` varchar(255) DEFAULT NULL,
`version` varchar(255) DEFAULT NULL,
`config` varchar(255) DEFAULT NULL,
`source` varchar(255) DEFAULT NULL,
`target` varchar(255) DEFAULT NULL,
`files` bigint(11) DEFAULT NULL,
`transfiles` bigint(11) DEFAULT NULL,
`bytes` bigint(11) DEFAULT NULL,
`transbytes` bigint(11) DEFAULT NULL,
`runtime` int(11) DEFAULT NULL,
`error` int(11) DEFAULT NULL,
`retries` int(11) DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
ENDOFHELP
exit;
}
############################################################################################################ END