From c83bc28475ee95df9912823ececf991a12065e8a Mon Sep 17 00:00:00 2001 From: Andreas Schmidt Date: Tue, 21 Apr 2015 23:31:56 +0200 Subject: [PATCH 1/4] Enable weekly, monthly, quarterly and yearly retention schema - new config properties EXPIRY_WEEK, EXPIRY_MONTH, EXPIRY_QUARTER and EXPIRY_YEAR - longest period wins (if first day of year is also first day of week year is selected) - support infinite retention with value 0 - explicitly use /bin/bash instead of /bin/sh to avoid script errors is bash is not default shell --- bin/add-host.sh | 2 +- bin/backup-runner.sh | 2 +- bin/backup.sh | 24 ++++++++++++++++++++---- bin/functions.sh | 2 +- bin/list-backups.sh | 4 ++-- bin/prune.sh | 2 +- etc/backup.conf | 6 +++++- 7 files changed, 31 insertions(+), 11 deletions(-) diff --git a/bin/add-host.sh b/bin/add-host.sh index c21447d..2e09d00 100755 --- a/bin/add-host.sh +++ b/bin/add-host.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Adlibre Backup - Add new host diff --git a/bin/backup-runner.sh b/bin/backup-runner.sh index 78e2dc9..894b4e2 100755 --- a/bin/backup-runner.sh +++ b/bin/backup-runner.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Adlibre Backup - Backup Runner - Backup Multiple hosts diff --git a/bin/backup.sh b/bin/backup.sh index 37c8864..4ddb382 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Adlibre Backup - Backup Single Host @@ -69,7 +69,20 @@ fi sourceHostConfig $HOSTS_DIR $HOST # Options Overridable by backup.conf (or command line) -EXPIRY=$(expr ${3-$EXPIRY} \* 24 \* 60 \* 60 + `date +%s`) # Convert expiry to unix epoc + +if [ -z "${3}" ]; then + if [ -v EXPIRY_WEEK -a "$(date +%u)" == "0" ]; then EXPIRY=${EXPIRY_WEEK}; fi + if [ -v EXPIRY_MONTH -a "$(date +%-d)" == "1" ]; then EXPIRY=${EXPIRY_MONTH}; fi + if [ -v EXPIRY_QUARTER -a "$(date +%-d)" == "1" -a "$(expr $(date '+%-m') % 3)" == "1" ]; then EXPIRY=${EXPIRY_QUARTER}; fi + if [ -v EXPIRY_YEAR -a "$(date +%-j)" == "1" ]; then EXPIRY=${EXPIRY_YEAR}; fi + echo "Expiry: $EXPIRY" +else + EXPIRY=${3} +fi + +if [ "${EXPIRY}" -gt "0" ]; then + EXPIRY=$(expr ${EXPIRY} \* 24 \* 60 \* 60 + `date +%s`) # Convert expiry to unix epoc +fi # Check to see if the host backup is disabled. if [ "${DISABLED}" == "true" ] && [ -z "$FORCE" ]; then @@ -82,14 +95,17 @@ eval "for e in $EXCLUDE $EXCLUDE_ADDITIONAL; do RSYNC_EXCLUDES=\"\$RSYNC_EXCLUDE # FIXME. Refactor do backup so we can properly handly dry-run if [ -n "$DRYRUN" ] ; then - echo "$DRYRUN Would have backed up $HOST." + echo "$DRYRUN Would have backed up $HOST with annotation ($ANNOTATION) and expiry ($EXPIRY)" exit fi # Do backup ( rm -f ${LOGFILE} # delete logfile from host dir before we begin. -echo $EXPIRY > ${HOSTS_DIR}${HOST}/c/EXPIRY +rm -f ${HOSTS_DIR}${HOST}/c/EXPIRY +if [ "${EXPIRY}" -gt "0" ]; then + echo $EXPIRY > ${HOSTS_DIR}${HOST}/c/EXPIRY +fi echo $ANNOTATION > ${HOSTS_DIR}${HOST}/c/ANNOTATION STARTTIME=$(date +%s) diff --git a/bin/functions.sh b/bin/functions.sh index f3a70fc..8d1e599 100644 --- a/bin/functions.sh +++ b/bin/functions.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Log Message to text file. diff --git a/bin/list-backups.sh b/bin/list-backups.sh index a272bdf..08aa424 100755 --- a/bin/list-backups.sh +++ b/bin/list-backups.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Adlibre Backup - List backups for a host @@ -34,7 +34,7 @@ for host in $HOSTS; do SNAPSHOT=$(basename $snapshot) EXPIRY=$(cat $snapshot/c/EXPIRY 2> /dev/null) ANNOTATION=$(cat $snapshot/c/ANNOTATION 2> /dev/null) - echo "$host $SNAPSHOT $EXPIRY \"$ANNOTATION\"" + echo "$host $SNAPSHOT ${EXPIRY:-0} \"$ANNOTATION\"" done fi done diff --git a/bin/prune.sh b/bin/prune.sh index 16dc93e..b21c411 100755 --- a/bin/prune.sh +++ b/bin/prune.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Adlibre Backup - Snapshot Pruner diff --git a/etc/backup.conf b/etc/backup.conf index 66b38d4..4a676e6 100644 --- a/etc/backup.conf +++ b/etc/backup.conf @@ -10,7 +10,11 @@ SSH_USER='root' EXCLUDE='/dev /proc /sys /tmp /var/tmp /var/run /selinux /cgroups lost+found' #BACKUP_PATHS='/etc /home' # BUG: FIXME, shell escaping issue causes multiple paths to fail. BACKUP_PATHS='/' -EXPIRY='28' # Default backup expiry (in days) +EXPIRY=$(expr 7 \* 8) # Default backup expiry (in days) +EXPIRY_WEEK=$(expr 7 \* 25) # twenty-five weeks (~3 month) +EXPIRY_MONTH=$(expr 2 \* 365) # two years +EXPIRY_QUARTER=$(expr 5 \* 365) # five years +EXPIRY_YEAR=0 # unlimited SSH_KEY=~root/.ssh/id_rsa.pub SNAPSHOT_ON_ERROR=false # Snapshot after rsync errors PRUNE=true # Prune old expired backup snapshots From d42c42ee98cdff616b5630527b200970660927a8 Mon Sep 17 00:00:00 2001 From: Andreas Schmidt Date: Wed, 22 Apr 2015 22:26:36 +0200 Subject: [PATCH 2/4] Some enhancements, esp. for retention policies - Section on retention policies in README.md - Ensure that special TTL only get applied to first (complete) backup of a given day - Introduce EXPIRY_DAY in order to be able to differentiate the (first) daily backup multiple backups at a given day. - Call bash via /usr/bin/env --- README.md | 35 +++++++++++++++++++++++++++++++++++ bin/add-host.sh | 2 +- bin/backup-runner.sh | 2 +- bin/backup.sh | 15 +++++++++------ bin/functions.sh | 2 +- bin/list-backups.sh | 4 ++-- bin/prune.sh | 2 +- bin/test-alerts.sh | 2 +- etc/backup.conf | 5 +++-- 9 files changed, 54 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0d4c3e7..d641d46 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,41 @@ Just dive in and copy the files out of the snapshot: cd /backup/hosts/example.com/.zfs/snapshot/ && \ rsync -aH --numeric-ids 2012-11-04-15:40:49-1352004049/d/ example.com:/restore-point/ + +## Retention Policies + +Each backup has a TTL defined, after which the prune script will delete the +backup's snapshot. You can configure the retention/expiry of backups through +various configuration properties. Each of them defines the TTL by means of "fully days after +backup creation". A value of 0 defines "keep forever". + +Once a backup is created you can't change the TTL anymore, as it is stored +within the read-only snapshot. + +A specific expiry TTL given as last parameter to the backup script +overrules any of the other TTL configuration. + +The default TTL configuration is set in the required property ``EXPIRY``. It gets +applied when none of the other option matches. + +All other options apply until to the first complete/successful backup at a given day: + +* ``EXPIRY_DAY``: TTL for daily backups +* ``EXPIRY_WEEK``: TTL for the backup on a Monday +* ``EXPIRY_MONTH``: TTL for backup on the first day of a month +* ``EXPIRY_QUARTER``: TTL for backup on the first day of a quarter +* ``EXPIRY_YEAR``: TTL: for backup on the first day of a year + +If a given day matches multiple conditions, the expiry configuration for the longest +period that has a non-empty value gets selected. E.g when on January 1st ``EXPIRY_YEAR`` +is not set, but ``EXPIRY_QUARTER`` and ``EXPIRY_DAY`` are, then +EXPRIY_QUARTER gets selected. + +When a backup doesn't succeed completely (partial backups), then the configured TTL +gets also applied to the next attempt to do a backup at the same day. If all backups +at a given day (partially) fail, then you won't get a backup with the special TTL. +You then would have to trigger a manual backup with a TTL set explicitly. + ## Status This should be considered "beta" status. diff --git a/bin/add-host.sh b/bin/add-host.sh index 2e09d00..98986ad 100755 --- a/bin/add-host.sh +++ b/bin/add-host.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Adlibre Backup - Add new host diff --git a/bin/backup-runner.sh b/bin/backup-runner.sh index 894b4e2..5d61729 100755 --- a/bin/backup-runner.sh +++ b/bin/backup-runner.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Adlibre Backup - Backup Runner - Backup Multiple hosts diff --git a/bin/backup.sh b/bin/backup.sh index 4ddb382..5952ea6 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Adlibre Backup - Backup Single Host @@ -71,11 +71,14 @@ sourceHostConfig $HOSTS_DIR $HOST # Options Overridable by backup.conf (or command line) if [ -z "${3}" ]; then - if [ -v EXPIRY_WEEK -a "$(date +%u)" == "0" ]; then EXPIRY=${EXPIRY_WEEK}; fi - if [ -v EXPIRY_MONTH -a "$(date +%-d)" == "1" ]; then EXPIRY=${EXPIRY_MONTH}; fi - if [ -v EXPIRY_QUARTER -a "$(date +%-d)" == "1" -a "$(expr $(date '+%-m') % 3)" == "1" ]; then EXPIRY=${EXPIRY_QUARTER}; fi - if [ -v EXPIRY_YEAR -a "$(date +%-j)" == "1" ]; then EXPIRY=${EXPIRY_YEAR}; fi - echo "Expiry: $EXPIRY" + NUM_BACKUPS_TODAY=$(find ${HOSTS_DIR}${HOST}/.${POOL_TYPE}/snapshot -maxdepth 1 -mindepth 1 -name "@$(date +%F)*" | grep -v -e "-partial$" | wc -l) + if [ "${NUM_BACKUPS_TODAY}" -eq "0" ]; then + if [ -n "${EXPIRY_DAY}" ]; then EXPIRY=${EXPIRY_DAY}; fi + if [ -n "${EXPIRY_WEEK}" -a "$(date +%u)" == "1" ]; then EXPIRY=${EXPIRY_WEEK}; fi + if [ -n "${EXPIRY_MONTH}" -a "$(date +%-d)" == "1" ]; then EXPIRY=${EXPIRY_MONTH}; fi + if [ -n "${EXPIRY_QUARTER}" -a "$(date +%-d)" == "1" -a "$(expr $(date '+%-m') % 3)" == "1" ]; then EXPIRY=${EXPIRY_QUARTER}; fi + if [ -n "${EXPIRY_YEAR}" -a "$(date +%-j)" == "1" ]; then EXPIRY=${EXPIRY_YEAR}; fi + fi else EXPIRY=${3} fi diff --git a/bin/functions.sh b/bin/functions.sh index 8d1e599..bb76410 100644 --- a/bin/functions.sh +++ b/bin/functions.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Log Message to text file. diff --git a/bin/list-backups.sh b/bin/list-backups.sh index 08aa424..29980bf 100755 --- a/bin/list-backups.sh +++ b/bin/list-backups.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # Adlibre Backup - List backups for a host CWD="$(dirname $0)/" # Source Config -. ${CWD}../etc/backup.conf +. ${CWD}../etc/backup.confgit # Source Functions . ${CWD}functions.sh; diff --git a/bin/prune.sh b/bin/prune.sh index b21c411..05a1251 100755 --- a/bin/prune.sh +++ b/bin/prune.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Adlibre Backup - Snapshot Pruner diff --git a/bin/test-alerts.sh b/bin/test-alerts.sh index ec029ff..a82499d 100755 --- a/bin/test-alerts.sh +++ b/bin/test-alerts.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Adlibre Backup - Test NSCA Alerts diff --git a/etc/backup.conf b/etc/backup.conf index 4a676e6..0b47e7b 100644 --- a/etc/backup.conf +++ b/etc/backup.conf @@ -10,8 +10,9 @@ SSH_USER='root' EXCLUDE='/dev /proc /sys /tmp /var/tmp /var/run /selinux /cgroups lost+found' #BACKUP_PATHS='/etc /home' # BUG: FIXME, shell escaping issue causes multiple paths to fail. BACKUP_PATHS='/' -EXPIRY=$(expr 7 \* 8) # Default backup expiry (in days) -EXPIRY_WEEK=$(expr 7 \* 25) # twenty-five weeks (~3 month) +EXPIRY=7 # Default backup expiry (in days) +EXPIRY_DAY=$(expr 7 \* 8) # 8 weeks (~2 month) +EXPIRY_WEEK=$(expr 7 \* 25) # twenty-five weeks (~6 month) EXPIRY_MONTH=$(expr 2 \* 365) # two years EXPIRY_QUARTER=$(expr 5 \* 365) # five years EXPIRY_YEAR=0 # unlimited From ab27ac3ad60c612cf88c8cbee1d8ede58062230d Mon Sep 17 00:00:00 2001 From: Andreas Schmidt Date: Sun, 4 Sep 2016 15:37:39 +0200 Subject: [PATCH 3/4] Fix typo on list-backups.sh --- bin/list-backups.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/list-backups.sh b/bin/list-backups.sh index 29980bf..9d08990 100755 --- a/bin/list-backups.sh +++ b/bin/list-backups.sh @@ -5,7 +5,7 @@ CWD="$(dirname $0)/" # Source Config -. ${CWD}../etc/backup.confgit +. ${CWD}../etc/backup.conf # Source Functions . ${CWD}functions.sh; From 6b119fc37a47c0a0995713e0214623eb0121dc38 Mon Sep 17 00:00:00 2001 From: Andreas Schmidt Date: Sun, 4 Sep 2016 17:57:53 +0200 Subject: [PATCH 4/4] Resolved stil plending merge conflict --- bin/backup.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bin/backup.sh b/bin/backup.sh index 692b7e3..957a06a 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -114,15 +114,11 @@ fi # Do backup ( rm -f ${LOGFILE} # delete logfile from host dir before we begin. -<<<<<<< HEAD +echo "inprogress" > $STATUSFILE rm -f ${HOSTS_DIR}${HOST}/c/EXPIRY if [ "${EXPIRY}" -gt "0" ]; then echo $EXPIRY > ${HOSTS_DIR}${HOST}/c/EXPIRY fi -======= -echo "inprogress" > $STATUSFILE -echo $EXPIRY > ${HOSTS_DIR}${HOST}/c/EXPIRY ->>>>>>> upstream/master echo $ANNOTATION > ${HOSTS_DIR}${HOST}/c/ANNOTATION STARTTIME=$(date +%s)