Skip to content

Commit 0b48f14

Browse files
perlDreamerperlDreamer
committedDec 13, 2011
Move to rsync based backup, both locally and remotely. Local only saves the last backup.
1 parent ca28628 commit 0b48f14

File tree

2 files changed

+42
-239
lines changed

2 files changed

+42
-239
lines changed
 

‎wre/sbin/backup.pl

+33-225
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use WRE::Mysql;
1919
use Getopt::Long ();
2020
use Pod::Usage ();
21+
use 5.010;
2122

2223
my $config = WRE::Config->new;
2324
my $util = WRE::File->new(wreConfig => $config);
@@ -32,74 +33,9 @@
3233
# are backups enabled
3334
exit unless $config->get("backup/enabled");
3435

35-
rotateBackupFiles($config);
3636
backupMysql($config);
37-
backupDomains($config);
38-
backupWebgui($config);
39-
backupWre($config);
40-
backupCustom($config);
4137
runExternalScripts($config);
42-
compressBackups($config);
4338
copyToRemote($config);
44-
#removeBackupFiles($config);
45-
46-
#-------------------------------------------------------------------
47-
sub backupCustom {
48-
my $config = shift;
49-
50-
return undef unless $config->get("backup/items/custom");
51-
52-
my $backupDir = dir($config->get("backup/path"));
53-
my $customFile = $config->getWebguiRoot("/sbin/preload.custom");
54-
open FILE, $customFile or die $!;
55-
my @lines = <FILE>;
56-
my @customDirs;
57-
foreach my $line (@lines) {
58-
if ($line =~ /^\//) {
59-
my $backupFile = join("",map { ucfirst($_) } split /\//, $line);
60-
chomp $backupFile;
61-
62-
eval { $util->tar(
63-
file => $backupDir->file($backupFile.".tar")->stringify,
64-
stuff => [$line],
65-
absPath => 1,
66-
)};
67-
print $@."\n" if ($@);
68-
}
69-
else {
70-
next;
71-
}
72-
}
73-
}
74-
75-
#-------------------------------------------------------------------
76-
sub backupDomains {
77-
my $config = shift;
78-
79-
# should we run?
80-
return undef unless $config->get("backup/items/domainsFolder");
81-
82-
my $domainsRoot = dir($config->getDomainRoot);
83-
84-
# get domains to backup
85-
opendir(DIR, $domainsRoot->stringify);
86-
my @domains = readdir(DIR);
87-
closedir(DIR);
88-
89-
# create tarballs
90-
my $tar = $config->get("tar");
91-
my $backupDir = dir($config->get("backup/path"));
92-
my $excludes = $config->getRoot("/etc/backup.exclude");
93-
foreach my $domain (@domains) {
94-
next if ($domain eq "." || $domain eq ".." || $domain eq "demo");
95-
eval {$util->tar(
96-
exclude => $excludes,
97-
file => $backupDir->file($domain.".tar")->stringify,
98-
stuff => [$domainsRoot->subdir($domain)->stringify],
99-
)};
100-
print $@."\n" if ($@);
101-
}
102-
}
10339

10440
#-------------------------------------------------------------------
10541
sub backupMysql {
@@ -143,183 +79,55 @@ sub backupMysql {
14379
}
14480

14581
#-------------------------------------------------------------------
146-
sub backupWebgui {
147-
my $config = shift;
148-
149-
# should we run?
150-
return undef unless $config->get("backup/items/webgui");
151-
152-
# create tarball
153-
eval {$util->tar(
154-
exclude => $config->getRoot("/etc/backup.exclude"),
155-
file => file($config->get("backup/path"), "webgui.tar")->stringify,
156-
stuff => [$config->getWebguiRoot],
157-
)};
158-
print $@."\n" if ($@);
159-
}
160-
161-
#-------------------------------------------------------------------
162-
sub backupWre {
163-
my $config = shift;
164-
my $full = $config->get("backup/items/fullWre");
165-
my $small = $config->get("backup/items/smallWre");
166-
167-
# should we run?
168-
return undef unless ($full || $small);
169-
170-
# whole thing or just configs?
171-
my $pathToBackup = ($full) ? $config->getRoot : $config->getRoot("/etc");
172-
173-
# create tarball
174-
eval {$util->tar(
175-
exclude => $config->getRoot("/etc/backup.exclude"),
176-
file => file($config->get("backup/path"), "wre.tar")->stringify,
177-
stuff => [$pathToBackup],
178-
)};
179-
print $@."\n" if ($@);
180-
}
181-
182-
#-------------------------------------------------------------------
183-
sub compressBackups {
82+
sub backupFiles {
18483
my $config = shift;
185-
my $gzip = file($config->get("gzip"))->stringify;
186-
my $backupDir = dir($config->get("backup/path"));
187-
188-
# compress files
189-
system("$gzip -f ".$backupDir->file("*.sql")->stringify);
190-
system("$gzip -f ".$backupDir->file("*.tar")->stringify);
84+
my $paths = $config->get("backup/items");
85+
my $backupDir = $config->get("backup/path");
86+
foreach my $path (@{ $paths }) {
87+
say "rsyncing $path locally...";
88+
system ("nice rsync -av --exclude=logs --exclude=mysqldata $path $backupDir/backup");
89+
}
19190
}
19291

19392
#-------------------------------------------------------------------
19493
sub copyToRemote {
19594
my $config = shift;
19695
# should we run?
197-
return undef unless $config->get("backup/ftp/enabled");
198-
my $protocol = $config->get("backup/ftp/protocol") || "ftp";
199-
200-
my $now = time;
201-
my $passive = $config->get("backup/ftp/usePassiveTransfers");
202-
my $host = $config->get("backup/ftp/hostname");
203-
my $path = $config->get("backup/ftp/path");
204-
my $user = $config->get("backup/ftp/user");
205-
my $pass = $config->get("backup/ftp/password");
206-
my $rotations = $config->get("backup/ftp/rotations");
207-
my $extraCommands = '';
208-
# don't validate local cert
209-
if ($protocol eq 'https') {
210-
$extraCommands .= 'set ssl:verify-certificate off; ';
211-
}
96+
return undef unless $config->get("backup/rsync/enabled");
21297

213-
# a little massage
214-
$path = ($path eq "/") ? "." : $path; # never let it look at the root of the server
98+
my $user = $config->get("backup/rsync/user");
99+
my $host = $config->get("backup/rsync/hostname");
100+
my $ACCOUNT = $user . '@' . $host;
101+
my $rotations = $config->get("backup/rsync/rotations");
102+
my $remote_path = $config->get("backup/rsync/remote_path");
103+
my $paths = $config->get("backup/items");
104+
my $backupDir = $config->get("backup/path");
215105

216106
# get old versions
217107
if ($rotations > 1) {
218-
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "cd '.$path.'; ls; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host.'/';
219-
my @dirs = ();
220-
if (open my $pipe, $cmd.'|') {
221-
while (my $line = <$pipe>) {
222-
chomp $line;
223-
if ($line =~ m/^([drxws-]+)\s+\d+\s+\w+\s+\w+\s+\d+\s+\w+\s+\d+\s+\d+:\d{2}\s+(\w+)/ || $line =~ m/^([drxws-]+)\s+--\s+(\w+)/) {
224-
my ($privs, $name) = ($1, $2);
225-
# skip non-backup directories
226-
next unless ($privs =~ m/^d/);
227-
next unless ($name =~ m/^\d+$/);
228-
push @dirs, $name;
229-
}
230-
}
231-
close $pipe;
108+
say "Removing last remote backup files...";
109+
my $cmd = "ssh $ACCOUNT rm -rf $remote_path/backup.$rotations";
110+
system($cmd);
111+
112+
# rotate backups except for the last one
113+
for (my $i=$rotations; $i > 1; $i--) {
114+
my $prev= $i - 1;
115+
say "Rotating old remote backup $i...";
116+
$cmd = "ssh $ACCOUNT mv $remote_path/backup.$prev $remote_path/backup.$i";
117+
system($cmd);
232118
}
233-
else {
234-
die "Couldn't connect to backup server.";
235-
}
236-
237-
# do rotations
238-
@dirs = sort(@dirs);
239-
my $i = scalar(@dirs);
240-
foreach my $dir (@dirs) {
241-
last if ($i < $rotations);
242-
system($config->getRoot('/prereqs/bin/lftp').' -e "rm -r -f '.$path.'/'.$dir.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host);
243-
$i--;
244-
}
245-
}
246-
247-
# deal with passive transfers
248-
if ($protocol eq 'ftp' && !$passive) {
249-
$extraCommands .= "set ftp:passive-mode off; ";
250-
}
251-
252-
# don't do the rotations folder if we aren't using rotations
253-
if ($rotations > 1) {
254-
$extraCommands .= 'mkdir '.$path.'/'.$now.'; mput -O '.$path.'/'.$now;
255-
}
256-
else {
257-
$extraCommands .= 'mput -O '.$path;
119+
say "Copying first backup to 1...";
120+
$cmd = "ssh $ACCOUNT cp -al $remote_path/backup $remote_path/backup.1";
121+
system($cmd);
258122
}
259123

260124
# do transfer
261-
my $cmd = $config->getRoot('/prereqs/bin/lftp').' -e "'.$extraCommands.' '.file($config->get("backup/path"),'/*.gz')->stringify.'; exit" -u '.$user.','.$pass.' '.$protocol.'://'.$host;
262-
system($cmd);
263-
}
264-
265-
#-------------------------------------------------------------------
266-
sub removeBackupFiles {
267-
my $config = shift;
268-
my $backupDir = dir($config->get("backup/path"));
269-
my $rotations = $config->get("backup/rotations");
270-
if ($rotations == 0 ||$rotations == 1) {
271-
opendir(DIR,$backupDir->stringify);
272-
my @files = readdir(DIR);
273-
closedir(DIR);
274-
foreach my $file (@files) {
275-
if ($rotations eq "0") {
276-
$backupDir->file($file)->remove;
277-
}
278-
elsif ($file =~ /(.*\.)1/ ) {
279-
$backupDir->file($file)->remove;
280-
}
281-
}
125+
say "Moving new data over...";
126+
foreach my $path (@{ $paths }) {
127+
say "rsyncing $path remotely...";
128+
system ("rsync -av --chmod=u+rwx --exclude=logs --exclude=mysqldata $path $ACCOUNT:$remote_path/backup");
282129
}
283130
}
284-
#-------------------------------------------------------------------
285-
sub rotateBackupFiles {
286-
my $config = shift;
287-
my $backupDir = dir($config->get("backup/path"));
288-
my $rotations = $config->get("backup/rotations") - 1; # .gz counts as 1
289-
290-
# get the list of files
291-
opendir(DIR,$backupDir->stringify);
292-
my @files = readdir(DIR);
293-
closedir(DIR);
294-
295-
# rotate and delete old files
296-
for (my $i = $rotations; $i > 0; $i--) {
297-
foreach my $file (@files) {
298-
299-
# only look at files where their extension is a specific digit
300-
if ($file =~ /(.*\.)$i$/) {
301-
my $filefront = $1;
302-
303-
# get rid of oldest files
304-
if ($i == $rotations) {
305-
$backupDir->file($file)->remove;
306-
}
307-
308-
# rotate younger files
309-
else {
310-
rename $backupDir->file($file)->stringify, $backupDir->file($filefront.($i+1))->stringify;
311-
}
312-
}
313-
}
314-
}
315-
316-
# rotate new files
317-
foreach my $file (@files) {
318-
if ($file =~ /\.gz$/) {
319-
rename $backupDir->file($file)->stringify, $backupDir->file($file.".1")->stringify;
320-
}
321-
}
322-
}
323131

324132
#-------------------------------------------------------------------
325133
sub runExternalScripts {

‎wre/var/setupfiles/wre.conf

+9-14
Original file line numberDiff line numberDiff line change
@@ -85,27 +85,22 @@
8585
"backup" : {
8686
"enabled" : 0,
8787
"path" : "/backup",
88-
"rotations" : 7,
8988
"mysql" : {
9089
"user" : "backup",
9190
"password" : "xxxxxxx"
9291
},
93-
"items" : {
94-
"mysql" : 1,
95-
"fullWre" : 0,
96-
"smallWre" : 1,
97-
"domainsFolder" : 1,
98-
"webgui" : 1
99-
},
92+
"items" : [ ##List of directories, wildcards are okay
93+
"/data/domains", ##All domain files
94+
"/data/WebGUI", ##All of WebGUI, including etc and sbin
95+
"/data/backup/*.sql.gz", ##All database dumps
96+
"/data/wre/etc" ##All WRE configuration files
97+
],
10098
"externalScripts" : [],
101-
"ftp" : {
99+
"rsync" : {
102100
"enabled" : 0,
103-
"usePassiveTransfers" : 1,
104101
"rotations" : 3,
105-
"hostname" : "ftp.example.com",
106-
"user" : "ftpuser",
107-
"protocol" : "ftp",
108-
"password" : "xxxxxx",
102+
"hostname" : "rsync.example.com",
103+
"user" : "rsyncuser",
109104
"path" : "."
110105
}
111106
},

0 commit comments

Comments
 (0)
Please sign in to comment.