diff --git a/.env.sample b/.env.sample index df7c075..67855c4 100644 --- a/.env.sample +++ b/.env.sample @@ -4,9 +4,9 @@ POSTGRES_PASSWORD=postgres S3_BACKUP_BUCKET= S3_BACKUP_PREFIX=qlockify -S3_BACKUP_REGION=us-east-1 S3_BACKUP_ENDPOINT_URL= S3_BACKUP_ACCESS_KEY_ID= S3_BACKUP_SECRET_ACCESS_KEY= BACKUP_ENCRYPTION_PASSPHRASE= -BACKUP_LOCAL_KEEP_LATEST=1 +BACKUP_LOCAL_KEEP_LATEST=3 +BACKUP_REMOTE_KEEP_LATEST=7 diff --git a/README.md b/README.md index c0025bb..4ff5d0f 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ Install backup upload prerequisites: ```bash sudo apt update -sudo apt install -y awscli openssl +sudo apt install -y rclone openssl ``` Configure encrypted S3-compatible backup uploads in `./.env`: @@ -275,21 +275,23 @@ Configure encrypted S3-compatible backup uploads in `./.env`: ```bash S3_BACKUP_BUCKET=qlockify-backups S3_BACKUP_PREFIX=qlockify -S3_BACKUP_REGION=us-east-1 S3_BACKUP_ENDPOINT_URL=https://c284984.parspack.net S3_BACKUP_ACCESS_KEY_ID= S3_BACKUP_SECRET_ACCESS_KEY= BACKUP_ENCRYPTION_PASSPHRASE= -BACKUP_LOCAL_KEEP_LATEST=1 +BACKUP_LOCAL_KEEP_LATEST=3 +BACKUP_REMOTE_KEEP_LATEST=7 ``` -Upload an encrypted backup to S3: +Upload an encrypted backup to S3-compatible object storage with rclone: ```bash chmod +x ./scripts/backup-upload-s3.sh ./scripts/backup-upload-s3.sh ``` +After a successful upload, the remote storage keeps only the latest `BACKUP_REMOTE_KEEP_LATEST` encrypted `.tar.gz.enc` files. The server keeps only the latest `BACKUP_LOCAL_KEEP_LATEST` plaintext `.tar.gz` files under `./backups/latest/`; encrypted backup files are not kept locally. + Restore the latest encrypted backup from S3: ```bash diff --git a/scripts/backup-upload-s3.sh b/scripts/backup-upload-s3.sh old mode 100644 new mode 100755 index 74c4745..672b2f4 --- a/scripts/backup-upload-s3.sh +++ b/scripts/backup-upload-s3.sh @@ -8,23 +8,25 @@ Usage: Environment: DEPLOY_ROOT Deployment directory. Defaults to ~/qlockify-deployment. - S3_BACKUP_BUCKET S3 bucket name. - S3_BACKUP_PREFIX S3 object prefix. Defaults to qlockify. - S3_BACKUP_REGION S3 region. Defaults to us-east-1. - S3_BACKUP_ENDPOINT_URL Optional S3-compatible endpoint URL. + + S3_BACKUP_ENDPOINT_URL S3 endpoint URL. S3_BACKUP_ACCESS_KEY_ID S3 access key. S3_BACKUP_SECRET_ACCESS_KEY S3 secret key. - BACKUP_ENCRYPTION_PASSPHRASE Passphrase used to encrypt backup archives. - BACKUP_LOCAL_KEEP_LATEST Set to 1 to keep latest encrypted archive locally. + S3_BACKUP_BUCKET Bucket name. + S3_BACKUP_PREFIX Object prefix. Defaults to qlockify. + + BACKUP_ENCRYPTION_PASSPHRASE Encryption passphrase. + BACKUP_LOCAL_KEEP_LATEST Number of latest local plaintext backups to keep. Defaults to 3. + BACKUP_REMOTE_KEEP_LATEST Number of latest remote encrypted backups to keep. Defaults to 7. EOF } log() { - printf '[backup-s3] %s\n' "$*" + printf '[backup-rclone] %s\n' "$*" } fail() { - printf '[backup-s3] %s\n' "$*" >&2 + printf '[backup-rclone] %s\n' "$*" >&2 exit 1 } @@ -35,26 +37,93 @@ require_var() { load_env() { local env_path="$1" + [[ -f "$env_path" ]] || fail "Deployment env file not found: $env_path" + set -a # shellcheck disable=SC1090 . "$env_path" set +a } -aws_s3() { - if [[ -n "${S3_BACKUP_ENDPOINT_URL:-}" ]]; then - aws --endpoint-url "$S3_BACKUP_ENDPOINT_URL" s3 "$@" +normalize_prefix() { + local value="${1:-qlockify}" + + value="${value#/}" + value="${value%/}" + + printf '%s' "$value" +} + +positive_integer_or_default() { + local value="${1:-}" + local fallback="$2" + + if [[ "$value" =~ ^[0-9]+$ ]]; then + printf '%s' "$value" else - aws s3 "$@" + printf '%s' "$fallback" fi } -normalize_prefix() { - local value="${1:-qlockify}" - value="${value#/}" - value="${value%/}" - printf '%s' "$value" +rclone_remote_path() { + printf 'parspack:%s/%s' "$S3_BACKUP_BUCKET" "$S3_BACKUP_PREFIX" +} + +cleanup_local_backups() { + local keep_count="$1" + local latest_dir="$2" + + [[ "$keep_count" -gt 0 ]] || { + rm -rf "$latest_dir" + return + } + + mkdir -p "$latest_dir" + find "$latest_dir" -maxdepth 1 -type f -name '*.tar.gz.enc' -delete + cp "$ARCHIVE_PATH" "$latest_dir/$ARCHIVE_NAME" + + find "$latest_dir" -maxdepth 1 -type f -name '*.tar.gz' -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep_count" 'NR > keep {print $2}' \ + | xargs -r rm -f + + log "Kept latest $keep_count plaintext backup(s) locally in: $latest_dir" +} + +cleanup_remote_backups() { + local keep_count="$1" + local remote_path="$2" + local stale_files + + [[ "$keep_count" -gt 0 ]] || { + log "Skipping remote cleanup because BACKUP_REMOTE_KEEP_LATEST is $keep_count" + return + } + + stale_files="$( + rclone lsf "$remote_path" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket \ + --files-only \ + | grep '\.tar\.gz\.enc$' \ + | sort -r \ + | awk -v keep="$keep_count" 'NR > keep' \ + || true + )" + + if [[ -z "$stale_files" ]]; then + log "No remote backups need cleanup" + return + fi + + while IFS= read -r stale_file; do + [[ -n "$stale_file" ]] || continue + log "Deleting old remote backup: $remote_path/$stale_file" + rclone deletefile "$remote_path/$stale_file" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket + done <<< "$stale_files" } if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then @@ -62,29 +131,28 @@ if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then exit 0 fi -command -v aws >/dev/null 2>&1 || fail "aws CLI is required" +command -v rclone >/dev/null 2>&1 || fail "rclone is required" command -v openssl >/dev/null 2>&1 || fail "openssl is required" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEPLOY_ROOT="${DEPLOY_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}" + DEPLOY_ENV="$DEPLOY_ROOT/.env" BACKUP_SCRIPT="$SCRIPT_DIR/backup.sh" [[ -x "$BACKUP_SCRIPT" ]] || fail "Backup script is not executable: $BACKUP_SCRIPT" + load_env "$DEPLOY_ENV" -S3_BACKUP_PREFIX="$(normalize_prefix "${S3_BACKUP_PREFIX:-qlockify}")" -S3_BACKUP_REGION="${S3_BACKUP_REGION:-us-east-1}" - -require_var S3_BACKUP_BUCKET +require_var S3_BACKUP_ENDPOINT_URL require_var S3_BACKUP_ACCESS_KEY_ID require_var S3_BACKUP_SECRET_ACCESS_KEY +require_var S3_BACKUP_BUCKET require_var BACKUP_ENCRYPTION_PASSPHRASE -export AWS_ACCESS_KEY_ID="$S3_BACKUP_ACCESS_KEY_ID" -export AWS_SECRET_ACCESS_KEY="$S3_BACKUP_SECRET_ACCESS_KEY" -export AWS_DEFAULT_REGION="$S3_BACKUP_REGION" -export AWS_EC2_METADATA_DISABLED=true +S3_BACKUP_PREFIX="$(normalize_prefix "${S3_BACKUP_PREFIX:-qlockify}")" +BACKUP_LOCAL_KEEP_LATEST="$(positive_integer_or_default "${BACKUP_LOCAL_KEEP_LATEST:-3}" 3)" +BACKUP_REMOTE_KEEP_LATEST="$(positive_integer_or_default "${BACKUP_REMOTE_KEEP_LATEST:-7}" 7)" WORK_DIR="$(mktemp -d)" trap 'rm -rf "$WORK_DIR"' EXIT @@ -93,36 +161,53 @@ PLAIN_DIR="$WORK_DIR/plain" mkdir -p "$PLAIN_DIR" log "Creating local backup archive" + DEPLOY_ROOT="$DEPLOY_ROOT" "$BACKUP_SCRIPT" "$PLAIN_DIR" shopt -s nullglob archives=("$PLAIN_DIR"/*.tar.gz) shopt -u nullglob + [[ "${#archives[@]}" -eq 1 ]] || fail "Expected exactly one backup archive, found ${#archives[@]}" ARCHIVE_PATH="${archives[0]}" ARCHIVE_NAME="$(basename "$ARCHIVE_PATH")" + ENCRYPTED_NAME="$ARCHIVE_NAME.enc" ENCRYPTED_PATH="$WORK_DIR/$ENCRYPTED_NAME" -S3_URI="s3://$S3_BACKUP_BUCKET/$S3_BACKUP_PREFIX/$ENCRYPTED_NAME" log "Encrypting backup archive" + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \ -in "$ARCHIVE_PATH" \ -out "$ENCRYPTED_PATH" \ -pass env:BACKUP_ENCRYPTION_PASSPHRASE -rm -f "$ARCHIVE_PATH" +RCLONE_CONFIG="$WORK_DIR/rclone.conf" -log "Uploading encrypted backup to $S3_URI" -aws_s3 cp "$ENCRYPTED_PATH" "$S3_URI" --only-show-errors +cat > "$RCLONE_CONFIG" <&2 + printf '[restore-rclone] %s\n' "$*" >&2 exit 1 } @@ -49,14 +48,6 @@ load_env() { set +a } -aws_s3() { - if [[ -n "${S3_BACKUP_ENDPOINT_URL:-}" ]]; then - aws --endpoint-url "$S3_BACKUP_ENDPOINT_URL" s3 "$@" - else - aws s3 "$@" - fi -} - normalize_prefix() { local value="${1:-qlockify}" value="${value#/}" @@ -64,12 +55,19 @@ normalize_prefix() { printf '%s' "$value" } -latest_object_key() { - aws_s3 ls "s3://$S3_BACKUP_BUCKET/$S3_BACKUP_PREFIX/" --recursive \ - | awk '{print $4}' \ +rclone_remote_path() { + printf 'parspack:%s/%s' "$S3_BACKUP_BUCKET" "$S3_BACKUP_PREFIX" +} + +latest_object_name() { + rclone lsf "$REMOTE_PATH" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket \ + --files-only \ | grep '\.tar\.gz\.enc$' \ | sort \ - | tail -n 1 + | tail -n 1 \ + || true } if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then @@ -83,7 +81,7 @@ REQUESTED_KEY="${1:-}" exit 1 } -command -v aws >/dev/null 2>&1 || fail "aws CLI is required" +command -v rclone >/dev/null 2>&1 || fail "rclone is required" command -v openssl >/dev/null 2>&1 || fail "openssl is required" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -95,35 +93,48 @@ RESTORE_SCRIPT="$SCRIPT_DIR/restore.sh" load_env "$DEPLOY_ENV" S3_BACKUP_PREFIX="$(normalize_prefix "${S3_BACKUP_PREFIX:-qlockify}")" -S3_BACKUP_REGION="${S3_BACKUP_REGION:-us-east-1}" require_var S3_BACKUP_BUCKET +require_var S3_BACKUP_ENDPOINT_URL require_var S3_BACKUP_ACCESS_KEY_ID require_var S3_BACKUP_SECRET_ACCESS_KEY require_var BACKUP_ENCRYPTION_PASSPHRASE -export AWS_ACCESS_KEY_ID="$S3_BACKUP_ACCESS_KEY_ID" -export AWS_SECRET_ACCESS_KEY="$S3_BACKUP_SECRET_ACCESS_KEY" -export AWS_DEFAULT_REGION="$S3_BACKUP_REGION" -export AWS_EC2_METADATA_DISABLED=true - -if [[ "$REQUESTED_KEY" == "latest" ]]; then - log "Resolving latest encrypted backup object" - OBJECT_KEY="$(latest_object_key)" - [[ -n "$OBJECT_KEY" ]] || fail "No encrypted backups found in s3://$S3_BACKUP_BUCKET/$S3_BACKUP_PREFIX/" -else - OBJECT_KEY="${REQUESTED_KEY#s3://$S3_BACKUP_BUCKET/}" -fi - WORK_DIR="$(mktemp -d)" trap 'rm -rf "$WORK_DIR"' EXIT -ENCRYPTED_PATH="$WORK_DIR/$(basename "$OBJECT_KEY")" +RCLONE_CONFIG="$WORK_DIR/rclone.conf" + +cat > "$RCLONE_CONFIG" <