#!/usr/bin/perl 
############################################################################################# mbackup (mwx'2023)
$VERSION='3.2.0';
use Getopt::Std;getopts('vVDWMYhsfc:');$opt_v=1 if ($opt_V);
use Net::Ping;
use Fcntl ':flock';
chomp($HOST=`hostname`);

&help if($opt_h);

$NOW=time();
my(@z)=localtime($NOW);$z[4]+=1;$z[5]-=100;$z[6]+=1;
$z=sprintf("%02d.%02d-%02d:%02d",$z[3],$z[4],$z[5],$z[2],$z[1]);


$MAX{'daily'}=7;                                                                                # default values
$MAX{'weekly'}=4;
$MAX{'monthly'}=12;
$MAX{'yearly'}=3;

$FLAT=0;
$RSYNCOPTS='--no-o --no-g --chmod=ugo=rwX';

$RSYNCRETRIES=3;

$CONFIG='';$REQUESTCONFIG=$ARGV[0];                                                         # load configuration

for ($opt_c,"/db/etc/mbackup.conf",$ENV{'HOME'}."/etc/mbackup.conf",$ENV{'HOME'}."/.mbackup.conf") {
  if (-f) {
    $confile=$_;
    last;
  }
}

require $confile;


for (keys(%GROUPS)) {                                                                         # build group list
  if ($_ eq $REQUESTCONFIG) {
    push(@GROUP,@{$GROUPS{$_}});
    $CONFIG=$REQUESTCONFIG;
  }
}


if ($CONFIG=~/^\s*$/) {                                                                 # check for valid config
  &log("ERROR: config '$REQUESTCONFIG' not found",1);
  exit;
}
$REMOTE=0;$REMOTE=1 if ($TARGETDIR!~/^\//);


if ($opt_s) {                                                                               # show configuration
  if (@GROUP) {
    print "\n";
    print "mode: group\n\n";
    print "group sleep: $GROUPSLEEP\n\n" if ($GROUPSLEEP>0);

    for $job (@GROUP) {
      print "/db/bin/mbackup -c $confile $job\n";
    }
    print "\n";
  } else {
    print "\n";
    if ($FLAT==1) {
      print "mode: flat\n";
    } else {
      print "mode: $MAX{'daily'} daily, $MAX{'weekly'} weekly, ";
      print "$MAX{'monthly'} monthly, $MAX{'yearly'} yearly\n";
    }
    if ($#EXCLUDE>-1) {
      print "\n";
      $tmp="";$tmp.="$_, " for (@EXCLUDE);$tmp=~s/\,\s+$//;
      print "exclude: $tmp\n";  
    }  
    print "\n";
    $l=0;
    for (keys(%SRC)) {
      $l=length($_) if length($_)>$l;
    }
    printf "%-${l}s -> %s\/%s\n",$_,$TARGETDIR,$SRC{$_} for (keys(%SRC));
    print "\n";  
  }
  exit;
}


if (@GROUP) {                                                                                 # run backup group
  for $job (@GROUP) {
    $cmd="/db/bin/mbackup -c $confile $job";

    &log("{$CONFIG} $cmd",1); 
    system($cmd);
    
    if ($GROUPSLEEP>0 && $job ne $GROUP[-1]) {
      &log("group loop sleep ($GROUPSLEEP seconds)",1); 
      sleep($GROUPSLEEP);
    }
     
  }
  exit;
}



if ($MY_LOG) {                                                                                    # mysql loging
  use DBI; 
  $dbh=DBI->connect("DBI:mysql:$MY_DB:$MY_HOST:",$MY_USER,$MY_PASSWD,{RaiseError=>1,PrintError=>1});
}


$mode='';                                                               # set mode (daily,weekly,monthly,yearly)
my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($NOW);
$mon++;$year+=1900;  
$z=sprintf("%02d\.%02d\.%04d %02d\:%02d",$mday,$mon,$year,$hour,$min);
$mode='daily'   if (             $MAX{'daily'}>0);
$mode='weekly'  if ( $wday==0 && $MAX{'weekly'}>0);
$mode='monthly' if ( $mday==1 && $MAX{'monthly'}>0);
$mode='yearly'  if ( $yday==0 && $MAX{'yearly'}>0);


$mode='daily' if ($opt_D);                                                                     # set mode manual
$mode='weekly' if ($opt_W);
$mode='monthly' if ($opt_M);
$mode='yearly' if ($opt_Y);

  
if (defined($PING)) {                                                               # check if host is reachable
  if (!Net::Ping->new("icmp", 15)->ping($PING)) {
     &log("ERROR: host $PING not reachable",1);exit;
  }
}

 
if (!-d "$TARGETDIR" && $REMOTE==0) {                                                         # check target dir
  if ($opt_f) {
    system("mkdir $TARGETDIR");
    if (!-d "$TARGETDIR") {
      &log("ERROR: target dir creation failed (mkdir $TARGETDIR)",1);
      exit;
    }
  } else {
    &log("ERROR: can not access target dir (mkdir $TARGETDIR)",1);
    exit;
  }
}   


for $srcdir (keys(%SRC)) {                                                                    # check source dir
  if ($srcdir=~/^\// && !-d $srcdir) {
    &log("ERROR: mising source dir '$srcdir'",1);
    exit;
  }
}


undef(%keycount);                                                              # check for duplicate source keys
for $srcdir (keys(%SRC)) {
  $keycount{$SRC{$srcdir}}++;
  if ($keycount{$SRC{$srcdir}}>1) {
    &log("ERROR: duplicate source key '$SRC{$srcdir}'",1);
    exit;
  }
}


if ($FLAT==1 || $REMOTE==1) {                                                                        # flat mode
  $toterr=0;$totfiles=0;$tottfi=0;$totbytes=0;$tottby=0;$totrt=0;

  for $srcdir (keys(%SRC)) {
    $subname=$SRC{$srcdir};

    $cmd="/usr/bin/rsync -av $RSYNCOPTS  --stats --delete " . 
         &exf() . "  --numeric-ids $srcdir/. $TARGETDIR/$subname";

    ($err,$fi,$tfi,$by,$tby,$rt,$try)=&rsync($cmd);
    
    &log("[$CONFIG] $srcdir,$TARGETDIR ($err,$tfi,$fi,$tby,$by,$rt)",1);
    
    if ($MY_LOG) {                                                                               # mysql logging
      $sql="INSERT INTO mbackup VALUES(0,now(),'$HOST','$VERSION','$CONFIG',".
           "'$srcdir','$TARGETDIR',$fi,$tfi,$by,$tby,$rt,$err,$try)";
      $dbh->do($sql);
    }
    
    if ($HTTP_LOG!~/^\s*$/) {                                                                     # http logging
      $tmp= "MBACKUP,$HOST,$VERSION,$CONFIG,$srcdir,$TARGETDIR,$fi,$tfi,$by,$tby,$rt,$err,$try";
      use LWP::UserAgent;
      my $ua = new LWP::UserAgent();
      $response = $ua->post("http://$HTTP_LOG", Content => $tmp);
    }

    $toterr+=$err;$totfi+=$fi;$tottfi+=$tfi;$totby+=$by;$tottby+=$tby;$totrt+=$rt;
    &log("rsync error $err in $srcdir/. -> $TARGETDIR/$subname") if ($err); 
  }
  if ($toterr==0) {
    &log("[$CONFIG] mbackup successful finished ($tottfi,$totfi,$tottby,$totby,$totrt)",1)   
  }
  exit;
}
  
  
if ($mode eq '') {                                                     # check if mode is defiend otherwise exit
  &log("ERROR: undefined mode",1);
  exit;  
}
  

$yweekly=0;$ymonthly=0;$yyearly=0;                                                 # search for missing snapshot
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
  my(@stat)=stat("$TARGETDIR/$file");

  if ($file=~/^weekly\.\d+$/) {
    $yweekly=$stat[9] if ($stat[9]>$yweekly);
  }
  if ($file=~/^monthly\.\d+$/) {
    $ymonthly=$stat[9] if ($stat[9]>$ymonthly);
  }
  if ($file=~/^yearly\.\d+$/) {
    $yyearly=$stat[9] if ($stat[9]>$yyearly);
  }
}
closedir(DIR);
if ($NOW-$yweekly>86400*10 && $yweekly!=0 && $MAX{'weekly'}>0) {
  &log("WARNING: no weekly snapshot for more than 10 days",1);
}
if ($NOW-$ymonthly>86400*40 && $ymonthly!=0 && $MAX{'monthly'}>0) {
  &log("WARNING: no monthly snapshot for more than 40 days",1);
}
if ($NOW-$yyearly>86400*400 && $yyearly!=0 && $MAX{'yearly'}>0) {
  &log("WARNING: no yearly snapshot for more than 400 days",1);
}


undef(%SNAPLIST);                                                          # read snapshot list for current mode
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
  my(@stat)=stat("$TARGETDIR/$file");
  $SNAPLIST{$1}=$stat[9] if ($file=~/^$mode\.(\d+)/);
}
closedir(DIR);


for $n (sort {$b<=>$a} keys(%SNAPLIST)) {                                                     # snapshot cycling
  $m=$n+1;
  if (-d "$TARGETDIR/$mode.$n" && !-e "$TARGETDIR/$mode.$m") {
      
    if ($m>=$MAX{$mode}) {
      &cmd("rm -rf $TARGETDIR/$mode.$n");
    } else {
      &cmd("mv $TARGETDIR/$mode.$n $TARGETDIR/$mode.$m");
    }
  } else {
    $tmp="";
    $tmp="($TARGETDIR/$mode.$n not found)" if (!-d "$TARGETDIR/$mode.$n");
    $tmp="($TARGETDIR/$mode.$m exists)" if (-e "$TARGETDIR/$mode.$m");
    &log("ERROR: error while snapshot cycling $tmp",1);exit;
  }
}


$youngest='';$y=0;                                                                # search for youngest snapshot
opendir(DIR,$TARGETDIR);
while ($file=readdir(DIR)) {
  my(@stat)=stat("$TARGETDIR/$file");
  if ($file=~/^((daily|weekly|monthly|yearly)\.\d+)/) {
    if ($stat[9]>$y) {
      $y=$stat[9];
      $youngest=$file;
    }
  }
}
closedir(DIR);


&cmd("mkdir -m 0755 -p $TARGETDIR/$mode.tmp/");                                             # create new $mode.0
&cmd("touch $TARGETDIR/$mode.tmp/");   


if (-d "$TARGETDIR/$mode.tmp") {                                                            # build new snapshot
  $toterr=0;$totfi=0;$tottfi=0;$totbytes=0;$tottby=0;

  for $srcdir (keys(%SRC)) {
    $subname=$SRC{$srcdir};

    $linkdest="";
    if ($youngest!~/^\s*$/) {
      if (-d "$TARGETDIR/$youngest" && -d "$TARGETDIR/$youngest/$subname") {
        $linkdest="--link-dest=$TARGETDIR/$youngest/$subname" 
      }
    }

    $cmd="/usr/bin/rsync -av $RSYNCOPTS --no-o --no-g --chmod=ugo=rwX --stats --delete " . 
         &exf() . " --numeric-ids $linkdest $srcdir/. $TARGETDIR/$mode.tmp/$subname";

    ($err,$fi,$tfi,$by,$tby,$rt,$try)=&rsync($cmd);

    &log("[$CONFIG] $srcdir,$TARGETDIR ($err,$tfi,$fi,$tby,$by,$rt)",1);
    
    if ($MY_LOG) {                                                                               # mysql logging
      $sql="INSERT INTO mbackup VALUES(0,now(),'$HOST','$VERSION','$CONFIG',".
           "'$srcdir','$TARGETDIR',$fi,$tfi,$by,$tby,$rt,$err,$try)";
      $dbh->do($sql);
    }
 
    if ($HTTP_LOG!~/^\s*$/) {                                                                     # http logging
      $tmp= "MBACKUP,$HOST,$VERSION,$CONFIG,$srcdir,$TARGETDIR,$fi,$tfi,$by,$tby,$rt,$err,$try";
      use LWP::UserAgent;
      my $ua = new LWP::UserAgent();
      $response = $ua->post("http://$HTTP_LOG", Content => $tmp);
    }


    $toterr+=$err;$totfi+=$fi;$tottfi+=$tfi;$totby+=$by;$tottby+=$tby;$totrt+=$rt;
  }
 
  if ($toterr==0) {

    &log("[$CONFIG] mbackup successful finished ($tottfi,$totfi,$tby,$by,$totrt)",1);
    &cmd("mv $TARGETDIR/$mode.tmp $TARGETDIR/$mode.0");
    
    utime $NOW,$NOW,"$TARGETDIR/$mode.0";
  }
      
} else {
  &log("ERROR: can not access $mode.tmp",1);
}   


######################################################################################################## SUPPORT

sub cmd() {                                                                                        # run command
  my($cmd)=@_;
  &log($cmd);
  system($cmd);
}

sub log() {                                                                                         # write logs
  my($str,$forceprint)=@_;  
  my(@z)=localtime(time());$z[4]+=1;$z[5]-=100;$z[6]+=1;
  $ts=sprintf("%02d\.%02d\.%02d %02d\:%02d",$z[3],$z[4],$z[5],$z[2],$z[1]);

  if ($LOGFILE!~/^\s+$/) {
    open(LOG,">> $LOGFILE");
    flock(LOG, LOCK_EX);
    print LOG "$ts $$/$VERSION/$CONFIG $str\n";
    close(LOG);
  }
  print "$str\n" if ($opt_v || $forceprint);
}

sub rsync() {                                                                                        # run rsync
  my($rsynccmd)=@_;
  
  my($files)=0;my($tfi)=0;my($bytes)=0;my($tby)=0;my($error)=0;my($try)=0;

  $runtime=time();
  $retries=0;
  
  for $try (1..$RSYNCRETRIES) {
    $rsynclog="";
  
    open(RSYNC,"$rsynccmd |");while(<RSYNC>) {
      print if ($opt_V);
      if (/Number of files:\s*([\d\,]+)/) {;$files=$1;$files=~s/,//g;}
      if (/Number of regular files transferred:\s*([\d\,]+)/) {;$tfi=$1;$tfi=~s/,//g;}
      if (/Total file size:\s*([\d\,]*)\s*bytes/) {;$bytes=$1;$bytes=~s/,//g;}
      if (/Total transferred file size:\s*([\d\,]*)\s*bytes/) {;$tby=$1;$tby=~s/,//g;}
    }
    close(RSYNC);
    $error=($? >> 8);
    $runtime=time()-$runtime;
  
    if ($error==0) {
      &log("$rsynccmd (successful)");
    } else {  
      &log("$rsynccmd (error $error in try $try)")
    } 

    last if ($error==0);

    $retries++;
  }

  return ($error,$files,$tfi,$bytes,$tby,$runtime,$retries);
}

sub exf() {                                                                                       # exclude file
  $ex='';
  if ($#EXCLUDE>-1) {
    $exfile="/tmp/mbackup.exclude.$CONFIG";
    open(OUT,"> $exfile");
    print OUT "$_\n" for (@EXCLUDE); 
    close(OUT);
    $ex="--exclude-from $exfile --delete-excluded";
  }
  return $ex;
}

sub buildsrc() {                                                            # build source list (used in config)
  my($base,@paths)=@_;
  for $path (@paths) {
    if ($path=~/\/([^\/]+)$/) {
      $SRC{"$base$path"}=$1;
    }
  }
}

sub sc() {                                                         # build simple configuration (used in config)
  my($uhd,$dirs,$exclude,$grp,$opt)=@_;
  
  $mode=0;
  if ($uhd=~/^([^\.]+)\.(.*)\:\:(.*)$/) {;$user=$3;$config=$1;$domain=$2;$mode=1;}
  if ($uhd=~/^(\w+)\@([^\.]+)\.(.*)$/) {;$user=$1;$config=$2;$domain=$3;$mode=2;}
  
  if ($mode>0) {
    %opt=%$opt;
    $config=$opt{name} if ($opt{name}); 

    @grp=@$grp;
    for $g (@grp) {
      push(@{$GROUPS{$g}},$config)  
    }
    
    if ($REQUESTCONFIG eq $config) {
      
      $RSYNCOPTS=$opt{rsync} if ($opt{rsync}); 
      $PING=$opt{ping} if ($opt{ping}); 
      $TARGETBASE=$opt{target} if ($opt{target}); 
    
      if ($opt{versions}=~/(\d+):(\d+):(\d+):(\d+)/) {
        $MAX{'daily'}=$1;
        $MAX{'weekly'}=$2;
        $MAX{'monthly'}=$3;
        $MAX{'yearly'}=$4;
        $FLAT=0;
      } else {
        $FLAT=1;
      }
    
      @DIRS=@$dirs;
      @EXCLUDE=@$exclude;
      $TARGETDIR="$TARGETBASE/$config";
      for $path (@DIRS) {
        $tpath=$path;$tpath=~s/^\s*\///;$tpath=~s/\//\./g;
        $SRC{"$uhd$path"}=$tpath if ($mode==1);
        $SRC{"$uhd:$path"}=$tpath if ($mode==2);
      }
    
      $CONFIG=$REQUESTCONFIG;
    }
  }
}

sub help() {                                                                                              # help
print "mbackup v$VERSION\n";
print <<'ENDOFHELP';

  USAGE:
  ------ 
    mbackup [-v] [-V] [-DWMY] <backup job name>
    mbackup -h
         
    -D   force daily mode
    -W   force weekly mode
    -M   force monthly mode
    -Y   force yearly mode
    -v   verbose output    
    -V   very verbose output
    -f   force creation of target directory
    -h   this help
           
  CONFIG FILE (/db/etc/mbackup.conf) GLOBAL OPTIONS:
  --------------------------------------------------
    $LOGFILE    = "/var/log/mbackup.log";      # logfile location
    
    $TARGETBASE = "/data/backup";              # targetbase for simple config

    $HTTP_LOG   = 'log.mxwx.org';              # http logging server

    $MY_LOG     = 1;                           # enable/disable mysql logging (deprecated)
    $MY_HOST    = 'localhost';                 
    $MY_USER    = '';
    $MY_PASSWD  = '';
    $MY_DB      = '';

  SIMPLE BACKUP JOB CONFIGURATION (/db/etc/mbackup.conf):
  -------------------------------------------------------

  &sc(target,[paths],[exclude],[groups],{ option=>value, .. }
  
  example:
  
    &sc('root@host.domain',['/etc','/var','/home'],['/lib/lxcfs'],['all']);
    &sc('host.domain::root',['/etc','/var','/home'],[],['all'], { rsync=>"-e 'ssh -p 2234'" });
    &sc('root@host.domain',['/etc','/var','/home'],[],[] { versions=>'7:4:12:3' });

  options:
  
    name    => 'name'                          # set configuration name
    rsync   => 'rsync options'                 # set additional rsync options  
    ping    => 'ip address'                    # apply ping test before backup
    version => 'daily:weekly:monthly:yearly'   # define backup versioning
    target  => '/path/to/mbackup'              # define backup destination

  BACKUP JOB CONFIGURATION (/db/etc/mbackup.conf):
  ------------------------------------------------
    if ($REQUESTCONFIG eq 'NAME') {            # basic backup job config
      <OPTIONS>
      $CONFIG=$REQUESTCONFIG;
    }
    
    if ($REQUESTCONFIG eq 'NAME') {            # backup group definition
      @GROUP=('NAME1','NAME2',...);
      $CONFIG=$REQUESTCONFIG;
    }

    push(@{$GROUPS{'all'}},'NAME');            # add config to group

    BACKUP JOB OPTIONS:
    -------------------
      # backup target definition (use only once)
      $TARGETDIR='/backup/NAME'                # local traget directory
      $TARGETDIR='10.0.1.1::root/backup/NAME'  # rsync protocol target
      $TARGETDIR='root@10.0.1.1:/backup/NAME'  # ssh traget directory

       
      # backup source definition (multiple allowd, different keys required)
      $SRC{'/etc'}               = 'etc1';     # local source
      $SRC{'10.0.1.1::root/etc'} = 'etc2';     # remote source (rsync protocol)
      $SRC{'root@10.0.1.1:/etc'} = 'etc3';     # remote source (ssh)
 
      # define eculsions (absolut path definitions from source path)
      @EXCLUDE=('/path1','/path2',...);  
 
      # define number of backups to keep, at least one value must be greater than 0
      $MAX{'daily'}   = 7;                     # keep n daily backup (default=7)
      $MAX{'weekly'}  = 4;                     # keep n weekly backup (default=4)
      $MAX{'monthly'} = 12;                    # keep n monthly backup (default=12)
      $MAX{'yearly'}  = 3;                     # keep n yearly backup (default=3)

      $FLAT=1;                                 # only keep one backup (default=0)
      $PING='10.0.1.1';                        # perform ping test before backup run 
      $RSYNCOPTS="-e 'ssh -p 2234'";           # add extra options for rsync
      $RSYNCRETRIES=3;                         # number of retries if rsync fails
      
  MYSQL LOGGING TABLE:
  --------------------
    CREATE TABLE `mbackup` (
      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
      `ts` timestamp NULL DEFAULT NULL,
      `host` varchar(255) DEFAULT NULL,
      `version` varchar(255) DEFAULT NULL,
      `config` varchar(255) DEFAULT NULL,
      `source` varchar(255) DEFAULT NULL,
      `target` varchar(255) DEFAULT NULL,
      `files` bigint(11) DEFAULT NULL,
      `transfiles` bigint(11) DEFAULT NULL,
      `bytes` bigint(11) DEFAULT NULL,
      `transbytes` bigint(11) DEFAULT NULL,
      `runtime` int(11) DEFAULT NULL,
      `error` int(11) DEFAULT NULL,
      `retries` int(11) DEFAULT '0',
      PRIMARY KEY (`id`)
   ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

ENDOFHELP
exit;
}

############################################################################################################ END