fix(deploy): wait for state stores on startup

docker-compose.prod.yml

@@ -1,23 +1,35 @@
 services:
   auth-service:
+    restart: unless-stopped
     environment:
       SMS_PROVIDER: ${SMS_PROVIDER:?set SMS_PROVIDER in production env}
       JWT_SECRET_KEY: ${JWT_SECRET_KEY:?set JWT_SECRET_KEY in production env}
       ADMIN_MOBILE: ${ADMIN_MOBILE:?set ADMIN_MOBILE in production env}
 
   sms-worker:
+    restart: unless-stopped
     environment:
       SMS_PROVIDER: ${SMS_PROVIDER:?set SMS_PROVIDER in production env}
       KAVENEGAR_API_KEY: ${KAVENEGAR_API_KEY:-}
       SMS_IR_API_KEY: ${SMS_IR_API_KEY:-}
 
   demo-app:
+    restart: unless-stopped
     environment:
       DEMO_ENABLE_DEBUG_OTP: "false"
     depends_on:
       auth-service:
         condition: service_healthy
 
+  mongo:
+    restart: unless-stopped
+
+  redis:
+    restart: unless-stopped
+
+  rabbitmq:
+    restart: unless-stopped
+
   caddy:
     image: caddy:2.8-alpine
     restart: unless-stopped
@@ -41,4 +53,3 @@ services:
 volumes:
   caddy-data:
   caddy-config:
-

docker-compose.yml

@@ -35,9 +35,9 @@ services:
       start_period: 20s
     depends_on:
       mongo:
-        condition: service_started
+        condition: service_healthy
       redis:
-        condition: service_started
+        condition: service_healthy
       rabbitmq:
         condition: service_healthy
 
@@ -54,6 +54,8 @@ services:
     depends_on:
       rabbitmq:
         condition: service_healthy
+      redis:
+        condition: service_healthy
 
   demo-app:
     build: .
@@ -72,15 +74,26 @@ services:
       auth-service:
         condition: service_healthy
       redis:
-        condition: service_started
+        condition: service_healthy
 
   mongo:
     image: mongo:7
     volumes:
       - mongo-data:/data/db
+    healthcheck:
+      test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping').ok"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
 
   redis:
     image: redis:7-alpine
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
 
   rabbitmq:
     image: rabbitmq:3.13-management-alpine

src/gapido_auth/infrastructure/container.py

@@ -1,3 +1,5 @@
+import asyncio
+import logging
 from dataclasses import dataclass
 from typing import Any
 
@@ -16,6 +18,10 @@ from gapido_auth.infrastructure.rabbitmq import RabbitMqSmsPublisher
 from gapido_auth.infrastructure.redis_otp_store import RedisOtpStore
 from gapido_auth.infrastructure.sms_provider import get_sms_template
 
+logger = logging.getLogger(__name__)
+STARTUP_RETRY_ATTEMPTS = 30
+STARTUP_RETRY_DELAY_SECONDS = 2
+
 
 @dataclass(slots=True)
 class AppContainer:
@@ -29,15 +35,17 @@ class AppContainer:
 async def build_auth_service(settings: Settings, rabbitmq_channel: AbstractChannel) -> AppContainer:
     """Create repositories, adapters, policies, and the AuthService use-case object."""
 
-    mongo_client: AsyncIOMotorClient[Any] = AsyncIOMotorClient(settings.mongo_uri)
+    mongo_client: AsyncIOMotorClient[Any] = AsyncIOMotorClient(
+        settings.mongo_uri,
+        serverSelectionTimeoutMS=2000,
+    )
     db = mongo_client[settings.mongo_db_name]
     users = MongoUserRepository(db)
     refresh_sessions = MongoRefreshSessionRepository(db)
-    await users.create_indexes()
-    await refresh_sessions.create_indexes()
-    await users.ensure_admin_user(settings.admin_mobile)
 
     redis = Redis.from_url(settings.redis_url, decode_responses=False)
+    await _wait_for_state_stores(users, refresh_sessions, redis, settings.admin_mobile)
+
     otp_store = RedisOtpStore(redis)
     sms_publisher = RabbitMqSmsPublisher(rabbitmq_channel)
     token_codec = JwtTokenCodec(
@@ -62,3 +70,30 @@ async def build_auth_service(settings: Settings, rabbitmq_channel: AbstractChann
         ),
     )
     return AppContainer(auth_service=auth_service, mongo_client=mongo_client, redis=redis)
+
+
+async def _wait_for_state_stores(
+    users: MongoUserRepository,
+    refresh_sessions: MongoRefreshSessionRepository,
+    redis: Redis,
+    admin_mobile: str,
+) -> None:
+    """Wait for MongoDB and Redis before exposing the gRPC server."""
+
+    for attempt in range(1, STARTUP_RETRY_ATTEMPTS + 1):
+        try:
+            await users.create_indexes()
+            await refresh_sessions.create_indexes()
+            await users.ensure_admin_user(admin_mobile)
+            await redis.ping()
+            return
+        except Exception:
+            if attempt >= STARTUP_RETRY_ATTEMPTS:
+                logger.exception("State stores did not become ready during auth-service startup")
+                raise
+            logger.info(
+                "Waiting for MongoDB/Redis before auth-service startup attempt=%s/%s",
+                attempt,
+                STARTUP_RETRY_ATTEMPTS,
+            )
+            await asyncio.sleep(STARTUP_RETRY_DELAY_SECONDS)