feat(worker): add configurable debug logging

backend/jobs/services.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from dataclasses import dataclass
 from datetime import timedelta
 
@@ -10,6 +11,8 @@ from django.utils import timezone
 
 from jobs.models import Job, JobEvent
 
+logger = logging.getLogger(__name__)
+
 
 class JobOwnershipLost(Exception):
     pass
@@ -67,6 +70,7 @@ def create_job(
     if normalized_key:
         existing = Job.objects.filter(idempotency_key=normalized_key).first()
         if existing is not None:
+            logger.info("Idempotent create returned existing job %s key=%s.", existing.id, normalized_key)
             return JobCreateResult(existing, created=False)
 
     try:
@@ -81,7 +85,9 @@ def create_job(
     except IntegrityError:
         if not normalized_key:
             raise
-        return JobCreateResult(Job.objects.get(idempotency_key=normalized_key), created=False)
+        existing = Job.objects.get(idempotency_key=normalized_key)
+        logger.info("Idempotent create raced and returned existing job %s key=%s.", existing.id, normalized_key)
+        return JobCreateResult(existing, created=False)
 
     emit_event(
         job,
@@ -94,6 +100,7 @@ def create_job(
             "idempotency_key": normalized_key,
         },
     )
+    logger.info("Created job %s type=%s priority=%s available_at=%s.", job.id, job.type, job.priority, job.available_at)
     return JobCreateResult(job, created=True)
 
 
@@ -114,6 +121,7 @@ def claim_next_job(*, worker_id: str, lease_seconds: int | None = None) -> Job |
 
     job = queryset.first()
     if job is None:
+        logger.debug("No claimable queued job found for worker=%s at %s.", worker_id, now.isoformat())
         return None
 
     job.status = Job.Status.RUNNING
@@ -130,6 +138,14 @@ def claim_next_job(*, worker_id: str, lease_seconds: int | None = None) -> Job |
         message="Job claimed",
         data={"locked_until": job.locked_until.isoformat()},
     )
+    logger.info(
+        "Claimed job %s type=%s attempt=%s worker=%s locked_until=%s.",
+        job.id,
+        job.type,
+        job.attempts,
+        worker_id,
+        job.locked_until,
+    )
     return job
 
 
@@ -145,6 +161,7 @@ def renew_lease(job_id, *, worker_id: str, attempt: int, lease_seconds: int | No
         attempts=attempt,
     ).update(locked_until=locked_until, updated_at=now)
     if not updated:
+        logger.debug("Lease renewal rejected job=%s worker=%s attempt=%s.", job_id, worker_id, attempt)
         return None
 
     job = Job.objects.get(id=job_id)
@@ -156,6 +173,7 @@ def renew_lease(job_id, *, worker_id: str, attempt: int, lease_seconds: int | No
         message="Lease renewed",
         data={"locked_until": locked_until.isoformat()},
     )
+    logger.debug("Renewed lease job=%s worker=%s attempt=%s locked_until=%s.", job_id, worker_id, attempt, locked_until)
     return job
 
 
@@ -172,6 +190,7 @@ def emit_progress(job_id, *, worker_id: str, attempt: int, percent: int, message
         .first()
     )
     if job is None:
+        logger.debug("Progress rejected job=%s worker=%s attempt=%s percent=%s.", job_id, worker_id, attempt, percent)
         return False
 
     emit_event(
@@ -182,6 +201,7 @@ def emit_progress(job_id, *, worker_id: str, attempt: int, percent: int, message
         message=message or f"{percent}% complete",
         data={"percent": percent},
     )
+    logger.debug("Recorded progress job=%s worker=%s attempt=%s percent=%s.", job_id, worker_id, attempt, percent)
     return True
 
 
@@ -202,6 +222,7 @@ def complete_job(job_id, *, worker_id: str, attempt: int, result: dict | None =
         updated_at=now,
     )
     if not updated:
+        logger.debug("Completion rejected job=%s worker=%s attempt=%s.", job_id, worker_id, attempt)
         return None
 
     job = Job.objects.get(id=job_id)
@@ -213,6 +234,7 @@ def complete_job(job_id, *, worker_id: str, attempt: int, result: dict | None =
         message="Job succeeded",
         data={"result": job.result},
     )
+    logger.info("Marked job %s succeeded worker=%s attempt=%s.", job.id, worker_id, attempt)
     return job
 
 
@@ -225,6 +247,7 @@ def fail_job(job_id, *, worker_id: str, attempt: int, error: str) -> Job | None:
         .first()
     )
     if job is None:
+        logger.debug("Failure rejected job=%s worker=%s attempt=%s error=%s.", job_id, worker_id, attempt, error)
         return None
 
     if job.attempts < job.max_attempts:
@@ -243,6 +266,15 @@ def fail_job(job_id, *, worker_id: str, attempt: int, error: str) -> Job | None:
             message=error,
             data={"delay_seconds": int(delay.total_seconds()), "available_at": job.available_at.isoformat()},
         )
+        logger.info(
+            "Scheduled retry job=%s attempt=%s/%s worker=%s delay_seconds=%s error=%s.",
+            job.id,
+            job.attempts,
+            job.max_attempts,
+            worker_id,
+            int(delay.total_seconds()),
+            error,
+        )
         return job
 
     job.status = Job.Status.FAILED
@@ -252,6 +284,7 @@ def fail_job(job_id, *, worker_id: str, attempt: int, error: str) -> Job | None:
     job.finished_at = now
     job.save(update_fields=["status", "locked_by", "locked_until", "last_error", "finished_at", "updated_at"])
     emit_event(job, JobEvent.Type.FAILED, attempt=attempt, worker_id=worker_id, message=error, data={"error": error})
+    logger.info("Marked job %s failed after attempt=%s worker=%s error=%s.", job.id, attempt, worker_id, error)
     return job
 
 
@@ -266,6 +299,7 @@ def cleanup_expired_jobs(*, batch_size: int | None = None) -> int:
         queryset = queryset.select_for_update()
 
     expired_jobs = list(queryset[:batch_size])
+    logger.debug("Found %s expired running job(s) for cleanup.", len(expired_jobs))
     for job in expired_jobs:
         worker_id = job.locked_by
         attempt = job.attempts
@@ -285,6 +319,14 @@ def cleanup_expired_jobs(*, batch_size: int | None = None) -> int:
                 message="Worker lease expired; job requeued",
                 data={"delay_seconds": int(delay.total_seconds()), "available_at": job.available_at.isoformat()},
             )
+            logger.info(
+                "Requeued expired job=%s attempt=%s/%s previous_worker=%s delay_seconds=%s.",
+                job.id,
+                attempt,
+                job.max_attempts,
+                worker_id,
+                int(delay.total_seconds()),
+            )
             continue
 
         job.status = Job.Status.FAILED
@@ -301,6 +343,7 @@ def cleanup_expired_jobs(*, batch_size: int | None = None) -> int:
             message="Worker lease expired; attempts exhausted",
             data={"error": "Worker lease expired"},
         )
+        logger.info("Failed expired job=%s attempt=%s previous_worker=%s.", job.id, attempt, worker_id)
 
     return len(expired_jobs)
 
@@ -333,6 +376,7 @@ def retry_failed_job(job_id) -> Job:
         ]
     )
     emit_event(job, JobEvent.Type.MANUAL_RETRY, message="Failed job manually retried")
+    logger.info("Manually retried failed job %s.", job.id)
     return job