Skip to content

Commit

Permalink
Move to rsync based backup, both locally and remotely. Local only sav…
Browse files Browse the repository at this point in the history 
…es the last backup.
  • Loading branch information
@perlDreamer
perlDreamer committed Dec 13, 2011
1 parent ca28628 commit 0b48f14
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 239 deletions.
258 changes: 33 additions & 225 deletions wre/sbin/backup.pl
View file
Expand Up @@ -18,6 +18,7 @@
use WRE::Mysql;
use Getopt::Long ();
use Pod::Usage ();
use 5.010;

my $config = WRE::Config->new;
my $util = WRE::File->new(wreConfig => $config);
Expand All @@ -32,74 +33,9 @@
# are backups enabled
exit unless $config->get("backup/enabled");

rotateBackupFiles($config);
backupMysql($config);
backupDomains($config);
backupWebgui($config);
backupWre($config);
backupCustom($config);
runExternalScripts($config);
compressBackups($config);
copyToRemote($config);
#removeBackupFiles($config);

#-------------------------------------------------------------------
sub backupCustom {
my $config = shift;

return undef unless $config->get("backup/items/custom");

my $backupDir = dir($config->get("backup/path"));
my $customFile = $config->getWebguiRoot("/sbin/preload.custom");
open FILE, $customFile or die $!;
my @lines = <FILE>;
my @customDirs;
foreach my $line (@lines) {
if ($line =~ /^\//) {
my $backupFile = join("",map { ucfirst($_) } split /\//, $line);
chomp $backupFile;

eval { $util->tar(
file => $backupDir->file($backupFile.".tar")->stringify,
stuff => [$line],
absPath => 1,
)};
print $@."\n" if ($@);
}
else {
next;
}
}
}

#-------------------------------------------------------------------
sub backupDomains {
my $config = shift;

# should we run?
return undef unless $config->get("backup/items/domainsFolder");

my $domainsRoot = dir($config->getDomainRoot);

# get domains to backup
opendir(DIR, $domainsRoot->stringify);
my @domains = readdir(DIR);
closedir(DIR);

# create tarballs
my $tar = $config->get("tar");
my $backupDir = dir($config->get("backup/path"));
my $excludes = $config->getRoot("/etc/backup.exclude");
foreach my $domain (@domains) {
next if ($domain eq "." || $domain eq ".." || $domain eq "demo");
eval {$util->tar(
exclude => $excludes,
file => $backupDir->file($domain.".tar")->stringify,
stuff => [$domainsRoot->subdir($domain)->stringify],
)};
print $@."\n" if ($@);
}
}

#-------------------------------------------------------------------
sub backupMysql {
Expand Down Expand Up @@ -143,183 +79,55 @@ sub backupMysql {
}

#-------------------------------------------------------------------
sub backupWebgui {
my $config = shift;

# should we run?
return undef unless $config->get("backup/items/webgui");

# create tarball
eval {$util->tar(
exclude => $config->getRoot("/etc/backup.exclude"),
file => file($config->get("backup/path"), "webgui.tar")->stringify,
stuff => [$config->getWebguiRoot],
)};
print $@."\n" if ($@);
}

#-------------------------------------------------------------------
sub backupWre {
my $config = shift;
my $full = $config->get("backup/items/fullWre");
my $small = $config->get("backup/items/smallWre");

# should we run?
return undef unless ($full || $small);

# whole thing or just configs?
my $pathToBackup = ($full) ? $config->getRoot : $config->getRoot("/etc");

# create tarball
eval {$util->tar(
exclude => $config->getRoot("/etc/backup.exclude"),
file => file($config->get("backup/path"), "wre.tar")->stringify,
stuff => [$pathToBackup],
)};
print $@."\n" if ($@);
}

#-------------------------------------------------------------------
sub compressBackups {
sub backupFiles {
my $config = shift;
my $gzip = file($config->get("gzip"))->stringify;
my $backupDir = dir($config->get("backup/path"));

# compress files
system("$gzip -f ".$backupDir->file("*.sql")->stringify);
system("$gzip -f ".$backupDir->file("*.tar")->stringify);
my $paths = $config->get("backup/items");
my $backupDir = $config->get("backup/path");
foreach my $path (@{ $paths }) {
say "rsyncing $path locally...";
system ("nice rsync -av --exclude=logs --exclude=mysqldata $path $backupDir/backup");
}
}

#-------------------------------------------------------------------
sub copyToRemote {
my $config = shift;
# should we run?
return undef unless $config->get("backup/ftp/enabled");
my $protocol = $config->get("backup/ftp/protocol") || "ftp";

my $now = time;
my $passive = $config->get("backup/ftp/usePassiveTransfers");
my $host = $config->get("backup/ftp/hostname");
my $path = $config->get("backup/ftp/path");
my $user = $config->get("backup/ftp/user");
my $pass = $config->get("backup/ftp/password");
my $rotations = $config->get("backup/ftp/rotations");
my $extraCommands = '';
# don't validate local cert
if ($protocol eq 'https') {
$extraCommands .= 'set ssl:verify-certificate off; ';
}
return undef unless $config->get("backup/rsync/enabled");

# a little massage
$path = ($path eq "/") ? "." : $path; # never let it look at the root of the server
my $user = $config->get("backup/rsync/user");
my $host = $config->get("backup/rsync/hostname");
my $ACCOUNT = $user . '@' . $host;
my $rotations = $config->get("backup/rsync/rotations");
my $remote_path = $config->get("backup/rsync/remote_path");
my $paths = $config->get("backup/items");
my $backupDir = $config->get("backup/path");

# get old versions
if ($rotations > 1) {
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "cd '.$path.'; ls; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host.'/';
my @dirs = ();
if (open my $pipe, $cmd.'|') {
while (my $line = <$pipe>) {
chomp $line;
if ($line =~ m/^([drxws-]+)\s+\d+\s+\w+\s+\w+\s+\d+\s+\w+\s+\d+\s+\d+:\d{2}\s+(\w+)/ || $line =~ m/^([drxws-]+)\s+--\s+(\w+)/) {
my ($privs, $name) = ($1, $2);
# skip non-backup directories
next unless ($privs =~ m/^d/);
next unless ($name =~ m/^\d+$/);
push @dirs, $name;
}
}
close $pipe;
say "Removing last remote backup files...";
my $cmd = "ssh $ACCOUNT rm -rf $remote_path/backup.$rotations";
system($cmd);

# rotate backups except for the last one
for (my $i=$rotations; $i > 1; $i--) {
my $prev= $i - 1;
say "Rotating old remote backup $i...";
$cmd = "ssh $ACCOUNT mv $remote_path/backup.$prev $remote_path/backup.$i";
system($cmd);
}
else {
die "Couldn't connect to backup server.";
}

# do rotations
@dirs = sort(@dirs);
my $i = scalar(@dirs);
foreach my $dir (@dirs) {
last if ($i < $rotations);
system($config->getRoot('/prereqs/bin/lftp').' -e "rm -r -f '.$path.'/'.$dir.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host);
$i--;
}
}

# deal with passive transfers
if ($protocol eq 'ftp' && !$passive) {
$extraCommands .= "set ftp:passive-mode off; ";
}

# don't do the rotations folder if we aren't using rotations
if ($rotations > 1) {
$extraCommands .= 'mkdir '.$path.'/'.$now.'; mput -O '.$path.'/'.$now;
}
else {
$extraCommands .= 'mput -O '.$path;
say "Copying first backup to 1...";
$cmd = "ssh $ACCOUNT cp -al $remote_path/backup $remote_path/backup.1";
system($cmd);
}

# do transfer
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "'.$extraCommands.' '.file($config->get("backup/path"),'/*.gz')->stringify.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host;
system($cmd);
}

#-------------------------------------------------------------------
sub removeBackupFiles {
my $config = shift;
my $backupDir = dir($config->get("backup/path"));
my $rotations = $config->get("backup/rotations");
if ($rotations == 0 ||$rotations == 1) {
opendir(DIR,$backupDir->stringify);
my @files = readdir(DIR);
closedir(DIR);
foreach my $file (@files) {
if ($rotations eq "0") {
$backupDir->file($file)->remove;
}
elsif ($file =~ /(.*\.)1/ ) {
$backupDir->file($file)->remove;
}
}
say "Moving new data over...";
foreach my $path (@{ $paths }) {
say "rsyncing $path remotely...";
system ("rsync -av --chmod=u+rwx --exclude=logs --exclude=mysqldata $path $ACCOUNT:$remote_path/backup");
}
}
#-------------------------------------------------------------------
sub rotateBackupFiles {
my $config = shift;
my $backupDir = dir($config->get("backup/path"));
my $rotations = $config->get("backup/rotations") - 1; # .gz counts as 1

# get the list of files
opendir(DIR,$backupDir->stringify);
my @files = readdir(DIR);
closedir(DIR);

# rotate and delete old files
for (my $i = $rotations; $i > 0; $i--) {
foreach my $file (@files) {

# only look at files where their extension is a specific digit
if ($file =~ /(.*\.)$i$/) {
my $filefront = $1;

# get rid of oldest files
if ($i == $rotations) {
$backupDir->file($file)->remove;
}

# rotate younger files
else {
rename $backupDir->file($file)->stringify, $backupDir->file($filefront.($i+1))->stringify;
}
}
}
}

# rotate new files
foreach my $file (@files) {
if ($file =~ /\.gz$/) {
rename $backupDir->file($file)->stringify, $backupDir->file($file.".1")->stringify;
}
}
}

#-------------------------------------------------------------------
sub runExternalScripts {
Expand Down
23 changes: 9 additions & 14 deletions wre/var/setupfiles/wre.conf
View file
Expand Up @@ -85,27 +85,22 @@
"backup" : {
"enabled" : 0,
"path" : "/backup",
"rotations" : 7,
"mysql" : {
"user" : "backup",
"password" : "xxxxxxx"
},
"items" : {
"mysql" : 1,
"fullWre" : 0,
"smallWre" : 1,
"domainsFolder" : 1,
"webgui" : 1
},
"items" : [ ##List of directories, wildcards are okay
"/data/domains", ##All domain files
"/data/WebGUI", ##All of WebGUI, including etc and sbin
"/data/backup/*.sql.gz", ##All database dumps
"/data/wre/etc" ##All WRE configuration files
],
"externalScripts" : [],
"ftp" : {
"rsync" : {
"enabled" : 0,
"usePassiveTransfers" : 1,
"rotations" : 3,
"hostname" : "ftp.example.com",
"user" : "ftpuser",
"protocol" : "ftp",
"password" : "xxxxxx",
"hostname" : "rsync.example.com",
"user" : "rsyncuser",
"path" : "."
}
},
Expand Down

0 comments on commit 0b48f14

Please sign in to comment.