From a617fdf1ecde63121d5e1a80cd92cc742d5aeace Mon Sep 17 00:00:00 2001 From: Amirhossein Khalili Date: Mon, 8 Jun 2026 20:01:43 +0330 Subject: [PATCH] feat(scripts): add scripts for backup/restore + upload to s3 storage --- .env.example | 11 ++ .gitattributes | 1 + .gitignore | 1 + README.md | 84 ++++++++++++++ scripts/backup-upload-s3.sh | 215 ++++++++++++++++++++++++++++++++++++ scripts/backup.sh | 127 +++++++++++++++++++++ scripts/restore-from-s3.sh | 153 +++++++++++++++++++++++++ scripts/restore.sh | 135 ++++++++++++++++++++++ 8 files changed, 727 insertions(+) create mode 100644 .gitattributes create mode 100644 scripts/backup-upload-s3.sh create mode 100644 scripts/backup.sh create mode 100644 scripts/restore-from-s3.sh create mode 100644 scripts/restore.sh diff --git a/.env.example b/.env.example index ea3f148..642cd69 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,13 @@ NEXT_HOST=east-guilan-ce.ir LETSENCRYPT_EMAIL=admin@east-guilan-ce.ir + +# Backups +S3_BACKUP_BUCKET=replace-with-bucket +S3_BACKUP_PREFIX=backup/guilan-ace +S3_BACKUP_REGION=us-east-1 +S3_BACKUP_ENDPOINT_URL=https://replace-with-s3-endpoint.example.com +S3_BACKUP_ACCESS_KEY_ID=replace-with-access-key +S3_BACKUP_SECRET_ACCESS_KEY=replace-with-secret-key +BACKUP_ENCRYPTION_PASSPHRASE=replace-with-long-random-passphrase +BACKUP_LOCAL_KEEP_LATEST=3 +BACKUP_REMOTE_KEEP_LATEST=10 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.gitignore b/.gitignore index 5edce2a..8bf273a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .env data/ +backups/ certs/fullchain.pem certs/privateKey.pem .vscode/ diff --git a/README.md b/README.md index 6048c47..6312c15 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,87 @@ parent-directory/ - The frontend is a Next.js standalone Node runtime behind Traefik on internal port `3000`. - Alertmanager has been removed from this stack. Prometheus scraping and Grafana provisioning remain intact. - Dockerfiles live only in this deployment repository, and compose is intentionally wired only for the nested production layout. + +## Backups and restores +The deployment repo owns backup automation in `scripts/`. Backups include: +- PostgreSQL dump from the `db` service. +- Django media files from the `django_media` Docker volume. +- Deployment, backend, and frontend `.env` files when present. + +The scripts expect this production layout: +```text +guilan-ace-deployment/ + .env + backend/guilan-ace-backend/.env + frontend/guilan-ace-frontend/.env + docker-compose.yml +``` + +Add these backup variables to `guilan-ace-deployment/.env`: +```env +S3_BACKUP_BUCKET=c284984 +S3_BACKUP_PREFIX=backup/guilan-ace +S3_BACKUP_REGION=us-east-1 +S3_BACKUP_ENDPOINT_URL=https://c284984.parspack.net +S3_BACKUP_ACCESS_KEY_ID=replace-with-access-key +S3_BACKUP_SECRET_ACCESS_KEY=replace-with-secret-key +BACKUP_ENCRYPTION_PASSPHRASE=replace-with-long-random-passphrase +BACKUP_LOCAL_KEEP_LATEST=3 +BACKUP_REMOTE_KEEP_LATEST=10 +``` + +Install script dependencies on the server: +```bash +sudo apt-get update +sudo apt-get install -y docker-compose-plugin rclone openssl +chmod +x ~/guilan-ace-deployment/scripts/*.sh +``` + +Create a manual local backup: +```bash +cd ~/guilan-ace-deployment +./scripts/backup.sh +``` + +Create, encrypt, upload, and rotate an S3 backup: +```bash +cd ~/guilan-ace-deployment +./scripts/backup-upload-s3.sh +``` + +Restore from a local backup archive: +```bash +cd ~/guilan-ace-deployment +./scripts/restore.sh backups/guilan-ace-backup-YYYYMMDD-HHMMSS.tar.gz +docker compose up -d --build +``` + +Restore the latest encrypted S3 backup: +```bash +cd ~/guilan-ace-deployment +./scripts/restore-from-s3.sh latest +docker compose up -d --build +``` + +Restore only selected parts by setting skip flags: +```bash +RESTORE_SKIP_ENV=1 ./scripts/restore.sh backups/guilan-ace-backup-YYYYMMDD-HHMMSS.tar.gz +RESTORE_SKIP_DB=1 RESTORE_SKIP_MEDIA=1 ./scripts/restore-from-s3.sh latest +``` + +Database restore recreates the configured database and is destructive unless `RESTORE_SKIP_DB=1` is set. + +Run encrypted S3 backups periodically with cron: +```bash +crontab -e +``` + +Example: run every day at `03:30` server time and write logs to `backups/backup.log`: +```cron +30 3 * * * cd /home/ubuntu/guilan-ace-deployment && ./scripts/backup-upload-s3.sh >> /home/ubuntu/guilan-ace-deployment/backups/backup.log 2>&1 +``` + +Example: run every 6 hours: +```cron +0 */6 * * * cd /home/ubuntu/guilan-ace-deployment && ./scripts/backup-upload-s3.sh >> /home/ubuntu/guilan-ace-deployment/backups/backup.log 2>&1 +``` diff --git a/scripts/backup-upload-s3.sh b/scripts/backup-upload-s3.sh new file mode 100644 index 0000000..96f164b --- /dev/null +++ b/scripts/backup-upload-s3.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: + backup-upload-s3.sh + +Environment: + DEPLOY_ROOT Deployment directory. Defaults to ~/guilan-ace-deployment. + + S3_BACKUP_ENDPOINT_URL S3 endpoint URL. + S3_BACKUP_ACCESS_KEY_ID S3 access key. + S3_BACKUP_SECRET_ACCESS_KEY S3 secret key. + S3_BACKUP_BUCKET Rclone bucket/root name, e.g. c284984. + S3_BACKUP_PREFIX Backup folder path. Defaults to backup/guilan-ace. + S3_BACKUP_REGION S3 region. Defaults to us-east-1. + + BACKUP_ENCRYPTION_PASSPHRASE Encryption passphrase. + BACKUP_LOCAL_KEEP_LATEST Number of latest local plaintext backups to keep. Defaults to 3. + BACKUP_REMOTE_KEEP_LATEST Number of latest remote encrypted backups to keep. Defaults to 7. +EOF +} + +log() { + printf '[backup-rclone] %s\n' "$*" +} + +fail() { + printf '[backup-rclone] %s\n' "$*" >&2 + exit 1 +} + +require_var() { + local name="$1" + [[ -n "${!name:-}" ]] || fail "$name is required" +} + +load_env() { + local env_path="$1" + + [[ -f "$env_path" ]] || fail "Deployment env file not found: $env_path" + + set -a + # shellcheck disable=SC1090 + . "$env_path" + set +a +} + +normalize_path() { + local value="${1:-backup/guilan-ace}" + + value="${value#/}" + value="${value%/}" + + printf '%s' "$value" +} + +positive_integer_or_default() { + local value="${1:-}" + local fallback="$2" + + if [[ "$value" =~ ^[0-9]+$ ]]; then + printf '%s' "$value" + else + printf '%s' "$fallback" + fi +} + +rclone_remote_path() { + printf 'parspack:%s/%s' "$S3_BACKUP_BUCKET" "$S3_BACKUP_PREFIX" +} + +cleanup_local_backups() { + local keep_count="$1" + local latest_dir="$2" + + [[ "$keep_count" -gt 0 ]] || { + rm -rf "$latest_dir" + return + } + + mkdir -p "$latest_dir" + find "$latest_dir" -maxdepth 1 -type f -name '*.tar.gz.enc' -delete + cp "$ARCHIVE_PATH" "$latest_dir/$ARCHIVE_NAME" + + find "$latest_dir" -maxdepth 1 -type f -name '*.tar.gz' -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep_count" 'NR > keep {print $2}' \ + | xargs -r rm -f + + log "Kept latest $keep_count plaintext backup(s) locally in: $latest_dir" +} + +cleanup_remote_backups() { + local keep_count="$1" + local remote_path="$2" + local stale_files + + [[ "$keep_count" -gt 0 ]] || { + log "Skipping remote cleanup because BACKUP_REMOTE_KEEP_LATEST is $keep_count" + return + } + + stale_files="$( + rclone lsf "$remote_path" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket \ + --files-only \ + | grep '\.tar\.gz\.enc$' \ + | sort -r \ + | awk -v keep="$keep_count" 'NR > keep' \ + || true + )" + + if [[ -z "$stale_files" ]]; then + log "No remote backups need cleanup" + return + fi + + while IFS= read -r stale_file; do + [[ -n "$stale_file" ]] || continue + log "Deleting old remote backup: $remote_path/$stale_file" + rclone deletefile "$remote_path/$stale_file" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket + done <<< "$stale_files" +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +command -v rclone >/dev/null 2>&1 || fail "rclone is required" +command -v openssl >/dev/null 2>&1 || fail "openssl is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="${DEPLOY_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}" + +DEPLOY_ENV="$DEPLOY_ROOT/.env" +BACKUP_SCRIPT="$SCRIPT_DIR/backup.sh" + +[[ -x "$BACKUP_SCRIPT" ]] || fail "Backup script is not executable: $BACKUP_SCRIPT" + +load_env "$DEPLOY_ENV" + +require_var S3_BACKUP_ENDPOINT_URL +require_var S3_BACKUP_ACCESS_KEY_ID +require_var S3_BACKUP_SECRET_ACCESS_KEY +require_var S3_BACKUP_BUCKET +require_var BACKUP_ENCRYPTION_PASSPHRASE + +S3_BACKUP_PREFIX="$(normalize_path "${S3_BACKUP_PREFIX:-backup/guilan-ace}")" +BACKUP_LOCAL_KEEP_LATEST="$(positive_integer_or_default "${BACKUP_LOCAL_KEEP_LATEST:-3}" 3)" +BACKUP_REMOTE_KEEP_LATEST="$(positive_integer_or_default "${BACKUP_REMOTE_KEEP_LATEST:-7}" 7)" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +PLAIN_DIR="$WORK_DIR/plain" +mkdir -p "$PLAIN_DIR" + +log "Creating local backup archive" + +DEPLOY_ROOT="$DEPLOY_ROOT" "$BACKUP_SCRIPT" "$PLAIN_DIR" + +shopt -s nullglob +archives=("$PLAIN_DIR"/*.tar.gz) +shopt -u nullglob + +[[ "${#archives[@]}" -eq 1 ]] || fail "Expected exactly one backup archive, found ${#archives[@]}" + +ARCHIVE_PATH="${archives[0]}" +ARCHIVE_NAME="$(basename "$ARCHIVE_PATH")" + +ENCRYPTED_NAME="$ARCHIVE_NAME.enc" +ENCRYPTED_PATH="$WORK_DIR/$ENCRYPTED_NAME" + +log "Encrypting backup archive" + +openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \ + -in "$ARCHIVE_PATH" \ + -out "$ENCRYPTED_PATH" \ + -pass env:BACKUP_ENCRYPTION_PASSPHRASE + +RCLONE_CONFIG="$WORK_DIR/rclone.conf" + +cat > "$RCLONE_CONFIG" <&2 + exit 1 +} + +require_file() { + local path="$1" + [[ -f "$path" ]] || fail "Required file not found: $path" +} + +compose() { + docker compose --env-file "$DEPLOY_ENV" -f "$DEPLOY_ROOT/docker-compose.yml" "$@" +} + +wait_for_db() { + compose exec -T db sh -c ' + set -eu + : "${DB_USER:?DB_USER is missing}" + : "${DB_NAME:?DB_NAME is missing}" + until pg_isready --username="$DB_USER" --dbname="$DB_NAME" --host=127.0.0.1; do + sleep 1 + done + ' >/dev/null +} + +copy_env_if_exists() { + local source_path="$1" + local target_path="$2" + + if [[ -f "$source_path" ]]; then + cp "$source_path" "$target_path" + fi +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +DEPLOY_ROOT="${DEPLOY_ROOT:-$HOME/guilan-ace-deployment}" +OUTPUT_DIR="${1:-$DEPLOY_ROOT/backups}" +BACKEND_ENV="$DEPLOY_ROOT/backend/guilan-ace-backend/.env" +FRONTEND_ENV="$DEPLOY_ROOT/frontend/guilan-ace-frontend/.env" +DEPLOY_ENV="$DEPLOY_ROOT/.env" + +require_file "$DEPLOY_ROOT/docker-compose.yml" +require_file "$DEPLOY_ENV" +require_file "$BACKEND_ENV" + +mkdir -p "$OUTPUT_DIR" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +TIMESTAMP="$(date -u +'%Y%m%d-%H%M%S')" +ARCHIVE_NAME="guilan-ace-backup-$TIMESTAMP.tar.gz" +ARCHIVE_PATH="$OUTPUT_DIR/$ARCHIVE_NAME" + +mkdir -p "$WORK_DIR/env" + +log "Checking Docker Compose configuration" +compose config -q + +log "Starting database service if needed" +compose up -d db +wait_for_db + +log "Dumping database from db service" +compose exec -T db sh -c ' + set -eu + : "${DB_USER:?DB_USER is missing}" + : "${DB_NAME:?DB_NAME is missing}" + pg_dump \ + --username="$DB_USER" \ + --dbname="$DB_NAME" \ + --format=plain \ + --clean \ + --if-exists \ + --no-owner \ + --no-privileges +' > "$WORK_DIR/database.sql" + +log "Archiving media files from Django media volume" +if compose ps --status running --services | grep -qx 'web'; then + compose exec -T web sh -c 'mkdir -p /app/media && tar -C /app/media -czf - .' > "$WORK_DIR/media.tar.gz" +else + log "Web service is not running; using a temporary container for media volume" + compose run --rm --no-deps --entrypoint sh web -c 'mkdir -p /app/media && tar -C /app/media -czf - .' > "$WORK_DIR/media.tar.gz" +fi + +log "Copying environment files" +copy_env_if_exists "$DEPLOY_ENV" "$WORK_DIR/env/deployment.env" +copy_env_if_exists "$BACKEND_ENV" "$WORK_DIR/env/backend.env" +copy_env_if_exists "$FRONTEND_ENV" "$WORK_DIR/env/frontend.env" + +cat > "$WORK_DIR/manifest.txt" < + +Examples: + restore-from-s3.sh latest + restore-from-s3.sh guilan-ace-backup-YYYYMMDD-HHMMSS.tar.gz.enc + +Environment: + DEPLOY_ROOT Deployment directory. Defaults to ~/guilan-ace-deployment. + S3_BACKUP_BUCKET Rclone bucket/root name, e.g. c284984. + S3_BACKUP_PREFIX Backup folder path. Defaults to backup/guilan-ace. + S3_BACKUP_ENDPOINT_URL S3-compatible endpoint URL. + S3_BACKUP_ACCESS_KEY_ID S3 access key. + S3_BACKUP_SECRET_ACCESS_KEY S3 secret key. + S3_BACKUP_REGION S3 region. Defaults to us-east-1. + BACKUP_ENCRYPTION_PASSPHRASE Passphrase used to decrypt backup archives. + +Existing RESTORE_SKIP_DB, RESTORE_SKIP_MEDIA, and RESTORE_SKIP_ENV flags are +passed through to restore.sh. +EOF +} + +log() { + printf '[restore-rclone] %s\n' "$*" +} + +fail() { + printf '[restore-rclone] %s\n' "$*" >&2 + exit 1 +} + +require_var() { + local name="$1" + [[ -n "${!name:-}" ]] || fail "$name is required" +} + +load_env() { + local env_path="$1" + [[ -f "$env_path" ]] || fail "Deployment env file not found: $env_path" + set -a + # shellcheck disable=SC1090 + . "$env_path" + set +a +} + +normalize_path() { + local value="${1:-backup/guilan-ace}" + value="${value#/}" + value="${value%/}" + printf '%s' "$value" +} + +rclone_remote_path() { + printf 'parspack:%s/%s' "$S3_BACKUP_BUCKET" "$S3_BACKUP_PREFIX" +} + +latest_object_name() { + rclone lsf "$REMOTE_PATH" \ + --config "$RCLONE_CONFIG" \ + --s3-no-check-bucket \ + --files-only \ + | grep '\.tar\.gz\.enc$' \ + | sort \ + | tail -n 1 \ + || true +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +REQUESTED_KEY="${1:-}" +[[ -n "$REQUESTED_KEY" ]] || { + usage + exit 1 +} + +command -v rclone >/dev/null 2>&1 || fail "rclone is required" +command -v openssl >/dev/null 2>&1 || fail "openssl is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="${DEPLOY_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}" +DEPLOY_ENV="$DEPLOY_ROOT/.env" +RESTORE_SCRIPT="$SCRIPT_DIR/restore.sh" + +[[ -x "$RESTORE_SCRIPT" ]] || fail "Restore script is not executable: $RESTORE_SCRIPT" +load_env "$DEPLOY_ENV" + +S3_BACKUP_PREFIX="$(normalize_path "${S3_BACKUP_PREFIX:-backup/guilan-ace}")" + +require_var S3_BACKUP_BUCKET +require_var S3_BACKUP_ENDPOINT_URL +require_var S3_BACKUP_ACCESS_KEY_ID +require_var S3_BACKUP_SECRET_ACCESS_KEY +require_var BACKUP_ENCRYPTION_PASSPHRASE + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +RCLONE_CONFIG="$WORK_DIR/rclone.conf" + +cat > "$RCLONE_CONFIG" < + +Environment: + DEPLOY_ROOT Deployment directory. Defaults to ~/guilan-ace-deployment. + RESTORE_SKIP_ENV Set to 1 to keep current .env files. + RESTORE_SKIP_MEDIA Set to 1 to keep current media files. + RESTORE_SKIP_DB Set to 1 to keep current database. + +This script restores: + - deployment, backend, and frontend .env files + - media files into the Django media volume + - PostgreSQL database from database.sql + +Database restore is destructive unless RESTORE_SKIP_DB=1 is set. +EOF +} + +log() { + printf '[restore] %s\n' "$*" +} + +fail() { + printf '[restore] %s\n' "$*" >&2 + exit 1 +} + +require_file() { + local path="$1" + [[ -f "$path" ]] || fail "Required file not found: $path" +} + +compose() { + docker compose --env-file "$DEPLOY_ENV" -f "$DEPLOY_ROOT/docker-compose.yml" "$@" +} + +wait_for_db() { + compose exec -T db sh -c ' + set -eu + : "${DB_USER:?DB_USER is missing}" + : "${DB_NAME:?DB_NAME is missing}" + until pg_isready --username="$DB_USER" --dbname="$DB_NAME" --host=127.0.0.1; do + sleep 1 + done + ' >/dev/null +} + +restore_env_if_present() { + local source_path="$1" + local target_path="$2" + + if [[ -f "$source_path" ]]; then + mkdir -p "$(dirname "$target_path")" + cp "$source_path" "$target_path" + chmod 600 "$target_path" || true + log "Restored $target_path" + fi +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +ARCHIVE_PATH="${1:-}" +[[ -n "$ARCHIVE_PATH" ]] || { + usage + exit 1 +} + +DEPLOY_ROOT="${DEPLOY_ROOT:-$HOME/guilan-ace-deployment}" +BACKEND_ENV="$DEPLOY_ROOT/backend/guilan-ace-backend/.env" +FRONTEND_ENV="$DEPLOY_ROOT/frontend/guilan-ace-frontend/.env" +DEPLOY_ENV="$DEPLOY_ROOT/.env" + +require_file "$ARCHIVE_PATH" +require_file "$DEPLOY_ROOT/docker-compose.yml" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +log "Extracting backup archive" +tar -C "$WORK_DIR" -xzf "$ARCHIVE_PATH" + +if [[ "${RESTORE_SKIP_ENV:-0}" != "1" ]]; then + log "Restoring environment files" + restore_env_if_present "$WORK_DIR/env/deployment.env" "$DEPLOY_ENV" + restore_env_if_present "$WORK_DIR/env/backend.env" "$BACKEND_ENV" + restore_env_if_present "$WORK_DIR/env/frontend.env" "$FRONTEND_ENV" +else + log "Skipping environment restore" +fi + +require_file "$BACKEND_ENV" +require_file "$DEPLOY_ENV" + +log "Checking Docker Compose configuration" +compose config -q + +if [[ "${RESTORE_SKIP_MEDIA:-0}" != "1" ]]; then + require_file "$WORK_DIR/media.tar.gz" + log "Restoring media files into Django media volume" + compose run --rm --no-deps --entrypoint sh web -c 'rm -rf /app/media/* /app/media/.[!.]* /app/media/..?* 2>/dev/null || true' + compose run --rm --no-deps -T --entrypoint sh web -c 'mkdir -p /app/media && tar -C /app/media -xzf -' < "$WORK_DIR/media.tar.gz" +else + log "Skipping media restore" +fi + +if [[ "${RESTORE_SKIP_DB:-0}" != "1" ]]; then + require_file "$WORK_DIR/database.sql" + log "Starting database service" + compose up -d db + wait_for_db + + log "Recreating and restoring database" + compose exec -T db sh -c ' + set -eu + : "${DB_USER:?DB_USER is missing}" + : "${DB_NAME:?DB_NAME is missing}" + psql --username="$DB_USER" --dbname=postgres --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '\''$DB_NAME'\'' AND pid <> pg_backend_pid();" >/dev/null + dropdb --username="$DB_USER" --if-exists "$DB_NAME" + createdb --username="$DB_USER" "$DB_NAME" + psql --username="$DB_USER" --dbname="$DB_NAME" + ' < "$WORK_DIR/database.sql" +else + log "Skipping database restore" +fi + +log "Restore completed" +log "Run: docker compose up -d --build"