diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5505067 --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Copy to .env on the server, replace every change-me value, then chmod 600 .env. +COMPOSE_PROJECT_NAME=zarinpal_analytics +CADDY_NETWORK=caddy_proxy + +# PostgreSQL. Keep the generated password URL-safe (for example: openssl rand -hex 32). +POSTGRES_DB=zarinpal +POSTGRES_USER=zarinpal +POSTGRES_PASSWORD=change-me-with-a-random-hex-value + +# Redis is private but authenticated. Keep this value URL-safe too. +REDIS_PASSWORD=change-me-with-a-different-random-hex-value + +# Absolute host path to the immutable first-import archive. +IMPORT_FILE=/srv/zarinpal-data/challenge_data.csv.gz + +# Optional backup settings. These are reserved for the backup/restore scripts. +S3_BACKUP_BUCKET= +S3_BACKUP_PREFIX=backup/zarinpal-analytics +S3_BACKUP_ENDPOINT_URL= +S3_BACKUP_ACCESS_KEY_ID= +S3_BACKUP_SECRET_ACCESS_KEY= +BACKUP_ENCRYPTION_PASSPHRASE= +BACKUP_LOCAL_KEEP_LATEST=3 +BACKUP_REMOTE_KEEP_LATEST=7 diff --git a/README.md b/README.md index ab0fb2c..25f34dc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository is the operational control plane for the hackathon project. It will own Docker Compose, Nginx, environment templates, deployment automation, backup/restore scripts, and cross-repository release orchestration. Product code belongs in the backend and frontend repositories. -Current state: specifications, operations plan, adaptation contract, runbooks, and task list are approved. Phase 4 deployment implementation is authorized. +The production stack is Docker Compose based and connects to the existing shared Caddy deployment. This repository does not publish host ports and does not run a second TLS proxy. For direct Windows development without Docker Compose, see [`docs/LOCAL-DEVELOPMENT.md`](docs/LOCAL-DEVELOPMENT.md). This helper is deliberately separate from the production Compose contract. @@ -16,7 +16,81 @@ zarinpal-analytics-deployment/ # this repository └── (deployment assets created through the authorized task list) ``` -The intended production hostname is `zarinpal.amiirkhl.ir`. The deployment shape will follow the proven Qlockify structure while being reviewed and hardened for this service; it is not a byte-for-byte copy. +The production hostname is `zarinpal.amiirkhl.ir`. Caddy routes `/api/*` and `/healthz` to the Go API and every other path to the static frontend. PostgreSQL, Redis, the worker, and all one-off commands remain on the private Compose data network. + +## Required server layout + +```text +/home/ubuntu/zarinpal-analytics-deployment/ +├── .env +├── backend/ +│ └── .env +└── frontend/ + └── .env +``` + +Clone the backend and frontend as ordinary ignored repositories, not submodules. Copy the three tracked examples to their ignored runtime names: + +```bash +cp .env.example .env +cp backend/.env.production.example backend/.env +cp frontend/.env.production.example frontend/.env +chmod 600 .env backend/.env frontend/.env +``` + +Replace every `change-me` value. Generate URL-safe secrets with `openssl rand -hex 32`. The root file owns the PostgreSQL and Redis passwords; Compose injects their connection settings into the Go containers. The frontend file contains public build-time switches only. + +## First production start + +Before adding/reloading the Caddy route, validate and build the stack, import the immutable dataset, create the demo account, and only then start the public upstreams: + +```bash +docker compose config --quiet +docker compose build backend frontend +docker compose up -d postgres redis +docker compose --profile tools run --rm migrate +docker compose --profile import run --rm importer +docker compose --profile tools run --rm admin +docker compose up -d backend worker frontend +docker compose ps +``` + +After every service reports healthy, validate and reload Caddy from the separate Caddy deployment. This order prevents the first public activation from serving an empty or partially imported database. + +## Normal later deployments + +Pull each repository manually, then run the guarded deploy script. It rejects dirty, non-`main`, or non-`origin/main` checkouts and never performs a fetch, reset, or checkout itself. + +```bash +git pull --ff-only origin main +git -C backend pull --ff-only origin main +git -C frontend pull --ff-only origin main +bash scripts/deploy.sh +``` + +Normal deploys run migrations but never run the importer or admin command. + +## Re-running one-off tools + +`IMPORT_FILE` in the root `.env` is the absolute host path of the immutable `challenge_data.csv.gz`. If recovery requires the idempotent importer or the demo account must be reset, run them explicitly: + +```bash +docker compose --profile import run --rm importer +docker compose --profile tools run --rm admin +``` + +Both commands are explicit one-offs. Re-running the importer relies on the backend's checksum-based idempotency guard. + +## Operations + +```bash +docker compose ps +docker compose logs --tail=200 backend worker frontend postgres redis +docker compose restart backend worker frontend +docker compose down +``` + +Do not use `docker compose down -v` in production; `-v` deletes the PostgreSQL and Redis volumes. Caddy is managed from its own repository and must already provide the external `caddy_proxy` network. ## Documentation workflow @@ -28,7 +102,7 @@ The intended production hostname is `zarinpal.amiirkhl.ir`. The deployment shape ## Platform operations documents -- `docs/PLATFORM-OPERATIONS-SPEC.md` defines the approved manual Qlockify-style production topology, lifecycle, observability phases, security, backup/restore, acceptance, and operator boundaries. +- `docs/PLATFORM-OPERATIONS-SPEC.md` defines the production topology and is amended by ADR 0006 for the shared Caddy ingress. - `docs/QLOCKIFY-ADAPTATION-CONTRACT.md` maps Qlockify's tracked deployment files and environment layout to the planned ZarinPal artifacts without requiring real server values during planning. - `docs/runbooks/FIRST-SERVER-BOOTSTRAP.md` defines the safe first-host sequence without implementing a bootstrap script. - `docs/runbooks/BACKUP-RESTORE.md` defines logical PostgreSQL backup, encryption/retention, guarded restore, and rehearsal. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..40b6cf2 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,197 @@ +name: ${COMPOSE_PROJECT_NAME:-zarinpal_analytics} + +x-logging: &bounded-logging + driver: json-file + options: + max-size: 10m + max-file: "3" + +x-backend-runtime: &backend-runtime + image: zarinpal-analytics-backend:${IMAGE_TAG:-local} + build: + context: ./backend + dockerfile: ../docker/backend.Dockerfile + env_file: + - ./backend/.env + environment: + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable + REDIS_ADDR: redis:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + init: true + restart: unless-stopped + security_opt: + - no-new-privileges:true + logging: *bounded-logging + +services: + postgres: + image: postgres:18.4-alpine3.24 + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql + networks: + - data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 10s + timeout: 3s + retries: 5 + start_period: 30s + shm_size: 128mb + mem_limit: 512m + cpus: 0.75 + pids_limit: 200 + logging: *bounded-logging + + redis: + image: redis:7.4.10-alpine3.21 + restart: unless-stopped + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + command: + - /bin/sh + - -ec + - | + umask 077 + printf 'appendonly yes\nappendfsync everysec\nmaxmemory 192mb\nmaxmemory-policy noeviction\nrequirepass %s\n' "$$REDIS_PASSWORD" > /tmp/redis.conf + chown redis:redis /data /tmp/redis.conf + exec su-exec redis redis-server /tmp/redis.conf + volumes: + - redis_data:/data + networks: + - data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + mem_limit: 256m + cpus: 0.25 + pids_limit: 100 + security_opt: + - no-new-privileges:true + logging: *bounded-logging + + backend: + <<: *backend-runtime + command: ["/app/api"] + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + expose: + - "8080" + networks: + data: + caddy_proxy: + aliases: + - zarinpal-backend + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/healthz"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 30s + stop_grace_period: 20s + mem_limit: 256m + cpus: 0.50 + pids_limit: 150 + + worker: + <<: *backend-runtime + command: ["/app/worker"] + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + networks: + - data + healthcheck: + test: ["CMD-SHELL", "kill -0 1"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 30s + stop_grace_period: 70s + mem_limit: 512m + cpus: 0.75 + pids_limit: 200 + + frontend: + image: zarinpal-analytics-frontend:${IMAGE_TAG:-local} + build: + context: ./frontend + dockerfile: ../docker/frontend.Dockerfile + init: true + restart: unless-stopped + expose: + - "8080" + networks: + caddy_proxy: + aliases: + - zarinpal-frontend + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/healthz"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 15s + security_opt: + - no-new-privileges:true + mem_limit: 96m + cpus: 0.25 + pids_limit: 100 + logging: *bounded-logging + + migrate: + <<: *backend-runtime + command: ["/app/migrate"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + networks: + - data + profiles: ["tools"] + + importer: + <<: *backend-runtime + command: ["/app/importer"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + volumes: + - ${IMPORT_FILE:-/srv/zarinpal-data/challenge_data.csv.gz}:/import/challenge_data.csv.gz:ro + networks: + - data + profiles: ["import"] + + admin: + <<: *backend-runtime + command: ["/app/admin"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + networks: + - data + profiles: ["tools"] + +networks: + data: + internal: true + caddy_proxy: + external: true + name: ${CADDY_NETWORK:-caddy_proxy} + +volumes: + postgres_data: + redis_data: diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile new file mode 100644 index 0000000..b380fec --- /dev/null +++ b/docker/backend.Dockerfile @@ -0,0 +1,28 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache ca-certificates git + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/api ./cmd/api \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/migrate ./cmd/migrate \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/importer ./cmd/importer \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/admin ./cmd/admin + +FROM alpine:3.22 + +RUN apk add --no-cache ca-certificates tzdata wget \ + && addgroup -S -g 10001 app \ + && adduser -S -D -H -u 10001 -G app app + +WORKDIR /app +COPY --from=builder /out/ /app/ +COPY --from=builder /src/migrations/ /app/migrations/ + +USER app:app +EXPOSE 8080 +CMD ["/app/api"] diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile new file mode 100644 index 0000000..bd851de --- /dev/null +++ b/docker/frontend.Dockerfile @@ -0,0 +1,41 @@ +FROM node:24-alpine AS builder + +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN npm run build + +FROM nginxinc/nginx-unprivileged:1.29-alpine + +COPY --from=builder /app/build /usr/share/nginx/html +RUN <<'EOF' cat > /etc/nginx/conf.d/default.conf +server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + index index.html; + server_tokens off; + + location = /healthz { + access_log off; + default_type text/plain; + return 200 "ok\n"; + } + + location ~* \.(?:css|js|woff2|png|jpg|jpeg|gif|svg|webp|ico)$ { + try_files $uri =404; + expires 1y; + add_header Cache-Control "public, max-age=31536000, immutable"; + } + + location / { + try_files $uri $uri/ /index.html; + add_header Cache-Control "no-cache"; + } +} +EOF + +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=3s --start-period=15s --retries=5 \ + CMD wget -qO- http://127.0.0.1:8080/healthz || exit 1 diff --git a/docs/PLATFORM-OPERATIONS-SPEC.md b/docs/PLATFORM-OPERATIONS-SPEC.md index 6960256..9de3a50 100644 --- a/docs/PLATFORM-OPERATIONS-SPEC.md +++ b/docs/PLATFORM-OPERATIONS-SPEC.md @@ -6,6 +6,8 @@ - Depends on: all deployable backend and frontend capability contracts - Last updated: 2026-08-20 +> **2026-08-22 ingress amendment:** ADR 0006 supersedes the dedicated Nginx/Certbot edge described below. The host's existing Caddy deployment owns ports 80/443, TLS, canonical-host routing, security headers, API/SSE proxying, and frontend proxying. This project publishes no ports and attaches only `backend` and `frontend` to the external `caddy_proxy` network. All private-data, manual-release, migration/import, secret, health, backup, and recovery requirements remain in force. + ## Confirmed intent Deploy `zarinpal.amiirkhl.ir` on one Ubuntu server using the proven Qlockify operating model: three nested Git repositories, Docker Compose builds performed on the server, Nginx as the only public entry point, automatic Let's Encrypt TLS, PostgreSQL 18, Redis 7 Alpine, a Go API and event-driven worker, and a static frontend. diff --git a/docs/QLOCKIFY-ADAPTATION-CONTRACT.md b/docs/QLOCKIFY-ADAPTATION-CONTRACT.md index cb916bf..63a82bf 100644 --- a/docs/QLOCKIFY-ADAPTATION-CONTRACT.md +++ b/docs/QLOCKIFY-ADAPTATION-CONTRACT.md @@ -2,6 +2,8 @@ - Status: Approved planning baseline - Source: `D:/Programing/Scripts/Qlockify/qlockify-deployment` (read-only) + +> **2026-08-22 ingress amendment:** ADR 0006 replaces the copied Nginx/Certbot edge with the host's existing Caddy deployment and external `caddy_proxy` network. Qlockify's nested-checkout, manual-build, environment, data-service, and operational patterns remain the baseline; its edge container and certificate lifecycle do not. - Target: `/home/ubuntu/zarinpal-analytics-deployment` - Implementation authorization: Granted by user on 2026-08-20 - Last updated: 2026-08-20 diff --git a/docs/decisions/0006-use-existing-caddy-ingress.md b/docs/decisions/0006-use-existing-caddy-ingress.md new file mode 100644 index 0000000..ea0a61c --- /dev/null +++ b/docs/decisions/0006-use-existing-caddy-ingress.md @@ -0,0 +1,21 @@ +# ADR 0006: Use the existing shared Caddy ingress + +- Status: Accepted +- Date: 2026-08-22 + +## Context + +The production host already runs a dedicated Caddy Compose project that owns ports 80/443, TLS certificates, and subdomain routing for Gitea, Penpot, Uptime Kuma, and other services. Running the previously planned Nginx/Certbot edge in this project would conflict on host ports and duplicate certificate ownership. + +## Decision + +ZarinPal Analytics connects its frontend and API containers to the existing external `caddy_proxy` Docker network using the stable aliases `zarinpal-frontend` and `zarinpal-backend`. Caddy routes `zarinpal.amiirkhl.ir` to those aliases. This Compose project publishes no host ports and owns no TLS material. + +PostgreSQL, Redis, the worker, migration, importer, and admin services remain on an internal data network. Only the API additionally joins `caddy_proxy`; the frontend joins only `caddy_proxy`. Caddy preserves the complete `/api` path and disables response buffering for proxied API traffic so authenticated SSE streams are delivered promptly. + +## Consequences + +- The Caddy deployment must exist first and create the external `caddy_proxy` network. +- Caddy configuration and this deployment must be released together for first activation. +- Nginx/Certbot-specific requirements in the earlier platform documents are superseded by this ADR. Their routing, TLS, and port-ownership intent is retained and implemented by the shared Caddy service. +- The shared proxy network is a trusted infrastructure boundary; no database, Redis, worker, or administrative service is attached to it. diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..4c1a7cd --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +DEPLOY_ROOT="${DEPLOY_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +BACKEND_ROOT="$DEPLOY_ROOT/backend" +FRONTEND_ROOT="$DEPLOY_ROOT/frontend" +COMPOSE=(docker compose --project-directory "$DEPLOY_ROOT" -f "$DEPLOY_ROOT/compose.yaml") + +fail() { + printf '[deploy] ERROR: %s\n' "$*" >&2 + exit 1 +} + +log() { + printf '[deploy] %s\n' "$*" +} + +check_repo() { + local label="$1" + local path="$2" + + [[ -d "$path/.git" ]] || fail "$label repository is missing at $path" + [[ "$(git -C "$path" branch --show-current)" == "main" ]] || fail "$label must be on main" + [[ -z "$(git -C "$path" status --porcelain)" ]] || fail "$label working tree is not clean" + git -C "$path" rev-parse --verify origin/main >/dev/null 2>&1 || fail "$label has no origin/main; pull it manually first" + [[ "$(git -C "$path" rev-parse HEAD)" == "$(git -C "$path" rev-parse origin/main)" ]] || fail "$label does not match origin/main; pull with --ff-only first" +} + +wait_healthy() { + local service="$1" + local deadline=$((SECONDS + 180)) + local container_id status + + while (( SECONDS < deadline )); do + container_id="$("${COMPOSE[@]}" ps -q "$service")" + if [[ -n "$container_id" ]]; then + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")" + [[ "$status" == "healthy" || "$status" == "running" ]] && return 0 + fi + sleep 3 + done + "${COMPOSE[@]}" ps + fail "$service did not become healthy" +} + +cd "$DEPLOY_ROOT" +[[ -f .env && -f backend/.env && -f frontend/.env ]] || fail "create .env, backend/.env, and frontend/.env first" +check_repo deployment "$DEPLOY_ROOT" +check_repo backend "$BACKEND_ROOT" +check_repo frontend "$FRONTEND_ROOT" +docker network inspect "${CADDY_NETWORK:-caddy_proxy}" >/dev/null 2>&1 || fail "the external Caddy network does not exist" + +IMAGE_TAG="$(git -C "$BACKEND_ROOT" rev-parse --short=12 HEAD)-$(git -C "$FRONTEND_ROOT" rev-parse --short=12 HEAD)" +export IMAGE_TAG +log "validating production configuration" +"${COMPOSE[@]}" config --quiet + +log "building application images" +"${COMPOSE[@]}" build backend frontend + +log "starting data services" +"${COMPOSE[@]}" up -d postgres redis +wait_healthy postgres +wait_healthy redis + +log "applying database migrations" +"${COMPOSE[@]}" --profile tools run --rm migrate + +log "starting API, worker, and frontend" +"${COMPOSE[@]}" up -d backend worker frontend +wait_healthy backend +wait_healthy worker +wait_healthy frontend + +log "deployment is healthy" +printf '[deploy] deployment=%s backend=%s frontend=%s image_tag=%s\n' \ + "$(git rev-parse HEAD)" \ + "$(git -C "$BACKEND_ROOT" rev-parse HEAD)" \ + "$(git -C "$FRONTEND_ROOT" rev-parse HEAD)" \ + "$IMAGE_TAG" +"${COMPOSE[@]}" ps