From dcce6d219dd947c17491e6cc68811b7b223068a1 Mon Sep 17 00:00:00 2001 From: Amirhossein Khalili Date: Tue, 15 Sep 2026 15:00:19 +0330 Subject: [PATCH] ci: deploy the verified local image Build and test the Next.js application once with persistent BuildKit caches, smoke-test the immutable runtime image, and activate that same image through the shared production Docker daemon with serialized rollback-safe deployment. --- .gitea/buildkitd.toml | 2 + .gitea/workflows/ci-cd.yml | 177 +++++++++++++++++++++---------------- DEPLOYMENT.md | 53 ++++++++--- Dockerfile | 33 ++++--- compose.build.yaml | 7 ++ compose.yaml | 21 +++-- scripts/deploy-release.sh | 91 ++++++++++++++----- scripts/smoke-container.sh | 65 ++++++++++++++ 8 files changed, 319 insertions(+), 130 deletions(-) create mode 100644 .gitea/buildkitd.toml create mode 100644 compose.build.yaml create mode 100644 scripts/smoke-container.sh diff --git a/.gitea/buildkitd.toml b/.gitea/buildkitd.toml new file mode 100644 index 0000000..f6dc454 --- /dev/null +++ b/.gitea/buildkitd.toml @@ -0,0 +1,2 @@ +[worker.oci] + max-parallelism = 1 diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 5238d00..1c5dc1b 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -12,68 +12,80 @@ on: jobs: verify: name: Verify production build - runs-on: ubuntu-latest + if: gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository + runs-on: portfolio-ci timeout-minutes: 15 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - - - name: Install dependencies - run: | - npm ci \ - --registry="https://package-mirror.liara.ir/repository/npm/" \ - --replace-registry-host=registry.npmjs.org \ - --include=optional - - - name: Lint - run: npm run lint - - - name: Type-check - run: npm run typecheck - - - name: Build standalone Next.js application - run: npm run build - - - name: Test standalone runtime - run: npm test - - - name: Audit production dependencies - run: | - npm audit \ - --registry="https://package-mirror.liara.ir/repository/npm/" \ - --audit-level=high - - deploy: - name: Deploy production - needs: verify - if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 20 env: - RELEASE_ID: ${{ gitea.sha }} - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - DEPLOY_USER: ${{ secrets.DEPLOY_USER }} - DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }} - DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }} - DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} - SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + BUILDER_NAME: portfolio-builder + VERIFY_IMAGE: portfolio-verify:${{ gitea.sha }} + RUNTIME_IMAGE: amiirkhl-portfolio:${{ gitea.sha }} + NPM_REGISTRY: https://package-mirror.liara.ir/repository/npm/ steps: - name: Check out repository uses: actions/checkout@v4 - - name: Configure SSH + - name: Prepare persistent BuildKit builder shell: bash + run: | + set -Eeuo pipefail + if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then + docker buildx create \ + --name "$BUILDER_NAME" \ + --driver docker-container \ + --buildkitd-config .gitea/buildkitd.toml + fi + docker buildx use "$BUILDER_NAME" + docker buildx inspect --bootstrap + + - name: Verify application once + shell: bash + run: | + set -Eeuo pipefail + docker buildx build \ + --builder "$BUILDER_NAME" \ + --target verification \ + --build-arg "NPM_REGISTRY=$NPM_REGISTRY" \ + --tag "$VERIFY_IMAGE" \ + --load \ + --progress plain \ + . + + - name: Audit dependencies + shell: bash + run: | + set -Eeuo pipefail + docker run --rm "$VERIFY_IMAGE" \ + npm audit \ + --registry="$NPM_REGISTRY" \ + --audit-level=high + + - name: Create immutable runtime image + shell: bash + run: | + set -Eeuo pipefail + docker buildx build \ + --builder "$BUILDER_NAME" \ + --target runtime \ + --build-arg "NPM_REGISTRY=$NPM_REGISTRY" \ + --tag "$RUNTIME_IMAGE" \ + --load \ + --progress plain \ + . + + - name: Smoke-test runtime image + shell: bash + run: bash scripts/smoke-container.sh "$RUNTIME_IMAGE" + + - name: Configure deployment SSH + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' + shell: bash + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }} run: | set -Eeuo pipefail : "${DEPLOY_HOST:?DEPLOY_HOST secret is required}" - : "${DEPLOY_USER:?DEPLOY_USER secret is required}" - : "${DEPLOY_PATH:?DEPLOY_PATH secret is required}" : "${DEPLOY_KNOWN_HOSTS:?DEPLOY_KNOWN_HOSTS secret is required}" : "${SSH_PRIVATE_KEY:?DEPLOY_SSH_KEY secret is required}" @@ -83,41 +95,56 @@ jobs: printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" chmod 600 "$HOME/.ssh/known_hosts" - - name: Package release - shell: bash - run: | - set -Eeuo pipefail - git archive \ - --format=tar.gz \ - --output="$RUNNER_TEMP/portfolio-$RELEASE_ID.tar.gz" \ - HEAD - - - name: Upload and activate release + - name: Upload release manifest + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' shell: bash + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} run: | set -Eeuo pipefail + : "${DEPLOY_USER:?DEPLOY_USER secret is required}" DEPLOY_PORT="${DEPLOY_PORT:-22}" - remote_archive="/tmp/portfolio-$RELEASE_ID.tar.gz" - ssh_options=( - -i "$HOME/.ssh/deploy_key" - -o BatchMode=yes - -o StrictHostKeyChecking=yes - -p "$DEPLOY_PORT" - ) - scp \ -i "$HOME/.ssh/deploy_key" \ -o BatchMode=yes \ -o StrictHostKeyChecking=yes \ -P "$DEPLOY_PORT" \ - "$RUNNER_TEMP/portfolio-$RELEASE_ID.tar.gz" \ - "$DEPLOY_USER@$DEPLOY_HOST:$remote_archive" + compose.yaml \ + "$DEPLOY_USER@$DEPLOY_HOST:/tmp/portfolio-${{ gitea.sha }}-compose.yaml" + - name: Activate verified image + if: gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' + shell: bash + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }} + DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + run: | + set -Eeuo pipefail + : "${DEPLOY_PATH:?DEPLOY_PATH secret is required}" + DEPLOY_PORT="${DEPLOY_PORT:-22}" + remote_compose="/tmp/portfolio-${{ gitea.sha }}-compose.yaml" printf -v remote_command \ - 'bash -s -- %q %q %q' \ - "$DEPLOY_PATH" "$RELEASE_ID" "$remote_archive" + 'bash -s -- %q %q %q %q %q' \ + "$DEPLOY_PATH" \ + "${{ gitea.sha }}" \ + "$RUNTIME_IMAGE" \ + "${{ gitea.run_number }}" \ + "$remote_compose" - ssh "${ssh_options[@]}" \ + ssh \ + -i "$HOME/.ssh/deploy_key" \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=yes \ + -p "$DEPLOY_PORT" \ "$DEPLOY_USER@$DEPLOY_HOST" \ "$remote_command" \ < scripts/deploy-release.sh + + - name: Remove temporary verification image + if: always() + shell: bash + run: docker image rm "$VERIFY_IMAGE" >/dev/null 2>&1 || true diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index f80f113..f0229a3 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,20 +1,27 @@ # Deployment The Gitea Actions workflow in `.gitea/workflows/ci-cd.yml` verifies every pull -request and every push to `main`. A successful push to `main` is then deployed -over SSH. The server builds the exact Git archive tested by CI, starts the -Compose service, and waits for its container health check. +request and every push to `main`. A successful push to `main` builds one +verified Docker image on the shared host daemon, smoke-tests it, and activates +that exact commit-tagged image over SSH without uploading source or rebuilding. -Releases are stored in `/releases/`. The `current` +Release manifests are stored in `/releases/`. Each +manifest records the immutable image tag and Gitea run number. The `current` symlink is changed only after the new container is healthy. If startup fails, -the deployment script automatically rebuilds and starts the previous release. +the deployment script restarts the previous image without rebuilding it. ## Gitea Actions setup -Enable **Repository Actions** in the repository settings and register a trusted -Gitea Runner. The workflow uses the `ubuntu-latest` label; that runner image must -provide Bash, Git, `tar`, OpenSSH (`ssh` and `scp`), and standard GNU utilities. -Node.js 22 is installed by the workflow. +Enable **Repository Actions** in the repository settings and register the +dedicated trusted runner maintained in +`https://git.amiirkhl.ir/amiirkhl/portfolio-gitea-runner`. The +workflow uses the `portfolio-ci` label. Its reusable job image provides Node.js +22, Bash, Git, OpenSSH, Docker CLI, Buildx, and Compose. Runner capacity remains +one because Gitea, builds, and production share a 2-core/4-GB server. + +The runner and job container use the host Docker socket. Restrict the label to +this trusted repository and disable Actions for fork pull requests. The workflow +also skips pull requests whose head repository differs from this repository. After the first successful verification run, protect `main` in Gitea and make the **Verify production build** status check required. This prevents a pull @@ -46,8 +53,9 @@ the protection against a man-in-the-middle attack. ## One-time server setup -The server needs Docker Engine with the Compose v2 plugin. The deployment user -must be able to run `docker` without an interactive password prompt. +The server needs Docker Engine with the Compose v2 and Buildx plugins. The +runner and deployment user must reach the same Docker daemon, and the deployment +user must run `docker` without an interactive password prompt. Create the deployment root and the external network once: @@ -88,9 +96,30 @@ optimizer-cache invalidation API. For an emergency manual deployment from a checkout on the server: ```sh -docker compose --project-name portfolio up --detach --build --remove-orphans +docker compose \ + --project-name portfolio \ + --file compose.yaml \ + --file compose.build.yaml \ + up --detach --build --remove-orphans ``` +This legacy path builds `amiirkhl-portfolio:latest` and is intentionally kept +outside the normal CI deployment. Existing source-based release directories can +still be started with their original Compose file during migration. Automatic +rollback begins with the first successful image-based release because older +release directories do not contain `release.env` image metadata. + +## Pipeline timing verification + +After migrating the runner, record one cold run and three normal source-change +runs. Expand the **Set up job** and **Complete job** sections in Gitea when +capturing the results. For each run, record checkout, verification build, audit, +runtime image creation, smoke test, deployment health wait, and completion time. +All three warm runs should finish below 180 seconds without OOM events or +production health failures. If they do not, collect `docker stats`, `free -h`, +`df -h`, `docker system df`, `docker buildx du --builder portfolio-builder`, and +the runner logs before changing or removing any quality gate. + Old versioned release directories are deliberately retained for rollback. They can be removed manually after a newer release has been proven stable; keep the directory referenced by the `current` symlink. diff --git a/Dockerfile b/Dockerfile index 0ba0247..8ffd02b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,37 +1,48 @@ -FROM node:22-alpine AS builder +# syntax=docker/dockerfile:1.7 + +FROM node:22-alpine AS dependencies WORKDIR /app ARG NPM_REGISTRY=https://package-mirror.liara.ir/repository/npm/ +ENV NEXT_TELEMETRY_DISABLED=1 COPY package.json package-lock.json ./ -RUN npm ci \ - --registry="${NPM_REGISTRY}" \ - --replace-registry-host=registry.npmjs.org \ - --include=optional +RUN --mount=type=cache,id=portfolio-npm-node22,target=/root/.npm,sharing=locked \ + npm ci \ + --registry="${NPM_REGISTRY}" \ + --replace-registry-host=registry.npmjs.org \ + --include=optional + +FROM dependencies AS verification COPY . . -RUN npm run build +RUN npm run lint +RUN npm run typecheck +RUN --mount=type=cache,id=portfolio-next-node22,target=/app/.next/cache,sharing=locked \ + npm run build +RUN npm test FROM node:22-alpine AS runtime WORKDIR /app ENV NODE_ENV=production \ + NEXT_TELEMETRY_DISABLED=1 \ HOSTNAME=0.0.0.0 \ PORT=3000 RUN addgroup --system --gid 1001 nodejs \ && adduser --system --uid 1001 nextjs -COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ -COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -COPY --from=builder --chown=nextjs:nodejs /app/public ./public +COPY --from=verification --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=verification --chown=nextjs:nodejs /app/.next/static ./.next/static +COPY --from=verification --chown=nextjs:nodejs /app/public ./public RUN mkdir -p .next/cache && chown -R nextjs:nodejs .next/cache USER nextjs EXPOSE 3000 -HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/ || exit 1 +HEALTHCHECK --interval=2s --timeout=2s --start-period=3s --retries=30 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] CMD ["node", "server.js"] diff --git a/compose.build.yaml b/compose.build.yaml new file mode 100644 index 0000000..fc138f7 --- /dev/null +++ b/compose.build.yaml @@ -0,0 +1,7 @@ +services: + portfolio-app: + image: amiirkhl-portfolio:latest + build: + context: . + dockerfile: Dockerfile + target: runtime diff --git a/compose.yaml b/compose.yaml index 74b07c4..0cdd29a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,17 +1,20 @@ services: portfolio-app: - image: amiirkhl-portfolio:latest - build: - context: . - dockerfile: Dockerfile + image: ${PORTFOLIO_IMAGE:-amiirkhl-portfolio:latest} container_name: portfolio-app restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/"] - interval: 30s - timeout: 3s - start_period: 10s - retries: 3 + test: + [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))", + ] + interval: 2s + timeout: 2s + start_period: 3s + retries: 30 volumes: - next-image-cache:/app/.next/cache networks: diff --git a/scripts/deploy-release.sh b/scripts/deploy-release.sh index 5afa366..89a43be 100644 --- a/scripts/deploy-release.sh +++ b/scripts/deploy-release.sh @@ -3,20 +3,32 @@ set -Eeuo pipefail deploy_root="${1:?deployment root is required}" release_id="${2:?release id is required}" -archive="${3:?release archive is required}" +image="${3:?runtime image is required}" +sequence="${4:?deployment sequence is required}" +incoming_compose="${5:?incoming Compose file is required}" if [[ "$deploy_root" != /* || "$deploy_root" == "/" ]]; then echo "DEPLOY_PATH must be an absolute directory other than /." >&2 exit 2 fi - if [[ ! "$release_id" =~ ^[0-9a-f]{7,64}$ ]]; then echo "The release id must be a Git commit SHA." >&2 exit 2 fi - -if [[ ! -f "$archive" ]]; then - echo "Release archive not found: $archive" >&2 +if [[ "$image" != "amiirkhl-portfolio:$release_id" ]]; then + echo "The runtime image must be tagged with the release id." >&2 + exit 2 +fi +if [[ ! "$sequence" =~ ^[1-9][0-9]*$ ]]; then + echo "The deployment sequence must be a positive integer." >&2 + exit 2 +fi +if [[ ! -f "$incoming_compose" ]]; then + echo "Incoming Compose file not found: $incoming_compose" >&2 + exit 2 +fi +if ! command -v flock >/dev/null 2>&1; then + echo "flock is required to serialize deployments." >&2 exit 2 fi @@ -26,7 +38,7 @@ current_link="$deploy_root/current" staging_dir="$releases_dir/.incoming-$release_id-$$" cleanup() { - rm -f -- "$archive" + rm -f -- "$incoming_compose" if [[ -d "$staging_dir" ]]; then rm -rf -- "$staging_dir" fi @@ -34,16 +46,12 @@ cleanup() { trap cleanup EXIT mkdir -p -- "$releases_dir" +exec 9>"$deploy_root/.deploy.lock" +flock -x 9 -if [[ ! -d "$release_dir" ]]; then - mkdir -- "$staging_dir" - tar -xzf "$archive" -C "$staging_dir" - mv -- "$staging_dir" "$release_dir" -fi - -if [[ ! -f "$release_dir/compose.yaml" ]]; then - echo "The release does not contain compose.yaml." >&2 - exit 2 +if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "Verified runtime image is missing from the production Docker daemon: $image" >&2 + exit 1 fi previous_release="" @@ -51,12 +59,41 @@ if [[ -L "$current_link" ]]; then previous_release="$(readlink -f -- "$current_link" || true)" fi +if [[ -n "$previous_release" && -f "$previous_release/release.env" ]]; then + current_sequence="$(sed -n 's/^DEPLOY_SEQUENCE=//p' "$previous_release/release.env")" + current_release="$(basename "$previous_release")" + if [[ "$current_release" == "$release_id" ]]; then + echo "Release is already active: $release_id" + exit 0 + fi + if [[ "$current_sequence" =~ ^[1-9][0-9]*$ ]] && (( sequence <= current_sequence )); then + echo "Refusing stale deployment sequence $sequence; current sequence is $current_sequence." >&2 + exit 1 + fi +fi + +if [[ ! -d "$release_dir" ]]; then + mkdir -- "$staging_dir" + install -m 600 "$incoming_compose" "$staging_dir/compose.yaml" + { + printf 'PORTFOLIO_IMAGE=%s\n' "$image" + printf 'DEPLOY_SEQUENCE=%s\n' "$sequence" + } > "$staging_dir/release.env" + mv -- "$staging_dir" "$release_dir" +fi + +if [[ ! -f "$release_dir/compose.yaml" || ! -f "$release_dir/release.env" ]]; then + echo "Release metadata is incomplete: $release_dir" >&2 + exit 2 +fi + compose() { local directory="$1" shift docker compose \ --project-name portfolio \ --file "$directory/compose.yaml" \ + --env-file "$directory/release.env" \ "$@" } @@ -73,7 +110,7 @@ wait_until_healthy() { for _ in {1..30}; do status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")" case "$status" in - healthy|running) + healthy) return 0 ;; unhealthy|exited|dead) @@ -88,27 +125,35 @@ wait_until_healthy() { return 1 } -deploy() { +activate() { local directory="$1" + local release_image + + release_image="$(sed -n 's/^PORTFOLIO_IMAGE=//p' "$directory/release.env")" + if [[ -z "$release_image" ]] || ! docker image inspect "$release_image" >/dev/null 2>&1; then + echo "Release image is unavailable: ${release_image:-}" >&2 + return 1 + fi + compose "$directory" config --quiet - compose "$directory" up --detach --build --remove-orphans + compose "$directory" up --detach --no-build --pull never --remove-orphans wait_until_healthy "$directory" } -if deploy "$release_dir"; then +if activate "$release_dir"; then new_link="$deploy_root/.current-$release_id-$$" ln -s -- "$release_dir" "$new_link" mv -Tf -- "$new_link" "$current_link" - echo "Deployment succeeded: $release_id" + echo "Deployment succeeded: $release_id ($image)" exit 0 fi echo "Deployment failed; attempting rollback." >&2 -if [[ -n "$previous_release" && -f "$previous_release/compose.yaml" ]]; then - deploy "$previous_release" +if [[ -n "$previous_release" && -f "$previous_release/compose.yaml" && -f "$previous_release/release.env" ]]; then + activate "$previous_release" echo "Rollback succeeded: $(basename "$previous_release")" >&2 else - echo "No previous release is available to roll back to." >&2 + echo "No image-based previous release is available for automatic rollback." >&2 fi exit 1 diff --git a/scripts/smoke-container.sh b/scripts/smoke-container.sh new file mode 100644 index 0000000..fba7935 --- /dev/null +++ b/scripts/smoke-container.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +image="${1:?runtime image is required}" +container_name="portfolio-smoke-${GITEA_RUN_ID:-$$}" + +cleanup() { + docker rm --force "$container_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +cleanup +docker run --detach --name "$container_name" "$image" >/dev/null + +for _ in {1..30}; do + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_name")" + case "$status" in + healthy) + break + ;; + unhealthy|exited|dead) + docker logs "$container_name" >&2 + exit 1 + ;; + esac + sleep 2 +done + +if [[ "$(docker inspect --format '{{.State.Health.Status}}' "$container_name")" != "healthy" ]]; then + docker logs "$container_name" >&2 + echo "Runtime image did not become healthy within 60 seconds." >&2 + exit 1 +fi + +docker exec -i "$container_name" node <<'NODE' +const origin = 'http://127.0.0.1:3000'; + +async function expectOk(path, options) { + const response = await fetch(`${origin}${path}`, options); + if (!response.ok) throw new Error(`${path} returned ${response.status}`); + return response; +} + +const english = await expectOk('/'); +await expectOk('/fa'); +await expectOk('/projects/interanet-solutions'); +await expectOk('/fa/projects/interanet-solutions'); + +const html = await english.text(); +const staticAsset = html.match(/(?:src|href)="(\/_next\/static\/[^"?#]+)/)?.[1]; +if (!staticAsset) throw new Error('No built static asset was found in the home page.'); +await expectOk(staticAsset); + +const optimized = await expectOk( + '/_next/image?url=%2Fimages%2Favatar.png&w=256&q=75', + { headers: { Accept: 'image/webp' } }, +); +const contentType = optimized.headers.get('content-type') ?? ''; +if (!/^image\/(?:webp|png)$/.test(contentType)) { + throw new Error(`Unexpected optimized image type: ${contentType}`); +} +if ((await optimized.arrayBuffer()).byteLength === 0) { + throw new Error('Optimized image response was empty.'); +} +NODE