Compare commits

...

3 Commits

12 changed files with 83 additions and 12 deletions

View File

@@ -133,6 +133,13 @@ $env:TEST_DATABASE_ENGINE="sqlite"
python -m pytest python -m pytest
``` ```
Run pytest with coverage:
```powershell
$env:TEST_DATABASE_ENGINE="sqlite"
python -m pytest --cov --cov-report=term-missing
```
Run worker locally: Run worker locally:
```powershell ```powershell

View File

@@ -1,4 +1,6 @@
DEBUG=true DEBUG=true
ENABLE_DJANGO_DEBUG_TOOLBAR=true
DJANGO_INTERNAL_IPS=127.0.0.1,localhost,host.docker.internal
DJANGO_SECRET_KEY=change-me DJANGO_SECRET_KEY=change-me
DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0
DJANGO_CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 DJANGO_CORS_ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173

View File

@@ -634,6 +634,8 @@ Example:
| Variable | Default | Purpose | | Variable | Default | Purpose |
| --- | --- | --- | | --- | --- | --- |
| `DEBUG` | `true` | Enables debug mode and permissive CORS in local development. | | `DEBUG` | `true` | Enables debug mode and permissive CORS in local development. |
| `ENABLE_DJANGO_DEBUG_TOOLBAR` | `true` when `DEBUG=true` | Enables Django Debug Toolbar for local backend requests. |
| `DJANGO_INTERNAL_IPS` | `127.0.0.1,localhost,host.docker.internal` | Internal IP allowlist used by Django Debug Toolbar. |
| `DJANGO_SECRET_KEY` | local dev key | Django secret key. Override outside local development. | | `DJANGO_SECRET_KEY` | local dev key | Django secret key. Override outside local development. |
| `DJANGO_ALLOWED_HOSTS` | `localhost,127.0.0.1,0.0.0.0` | Allowed hosts. | | `DJANGO_ALLOWED_HOSTS` | `localhost,127.0.0.1,0.0.0.0` | Allowed hosts. |
| `DJANGO_CORS_ALLOWED_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Frontend origins. | | `DJANGO_CORS_ALLOWED_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Frontend origins. |
@@ -1012,6 +1014,13 @@ cd backend
$env:TEST_DATABASE_ENGINE='sqlite'; .\.venv\Scripts\python.exe -m pytest $env:TEST_DATABASE_ENGINE='sqlite'; .\.venv\Scripts\python.exe -m pytest
``` ```
Run backend tests with coverage:
```powershell
cd backend
$env:TEST_DATABASE_ENGINE='sqlite'; .\.venv\Scripts\python.exe -m pytest --cov --cov-report=term-missing
```
The test suite covers: The test suite covers:
- successful claim - successful claim

View File

@@ -10,6 +10,9 @@ BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "django-insecure-job-queue-local-dev-key") SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "django-insecure-job-queue-local-dev-key")
DEBUG = os.getenv("DEBUG", "true").lower() in {"1", "true", "yes"} DEBUG = os.getenv("DEBUG", "true").lower() in {"1", "true", "yes"}
ALLOWED_HOSTS = [host.strip() for host in os.getenv("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,0.0.0.0").split(",")] ALLOWED_HOSTS = [host.strip() for host in os.getenv("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,0.0.0.0").split(",")]
ENABLE_DJANGO_DEBUG_TOOLBAR = (
DEBUG and os.getenv("ENABLE_DJANGO_DEBUG_TOOLBAR", "true").lower() in {"1", "true", "yes"}
)
INSTALLED_APPS = [ INSTALLED_APPS = [
"unfold", "unfold",
@@ -25,6 +28,9 @@ INSTALLED_APPS = [
"jobs", "jobs",
] ]
if ENABLE_DJANGO_DEBUG_TOOLBAR:
INSTALLED_APPS.append("debug_toolbar")
MIDDLEWARE = [ MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware", "django.middleware.security.SecurityMiddleware",
"corsheaders.middleware.CorsMiddleware", "corsheaders.middleware.CorsMiddleware",
@@ -36,6 +42,9 @@ MIDDLEWARE = [
"django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware",
] ]
if ENABLE_DJANGO_DEBUG_TOOLBAR:
MIDDLEWARE.insert(2, "debug_toolbar.middleware.DebugToolbarMiddleware")
ROOT_URLCONF = "config.urls" ROOT_URLCONF = "config.urls"
TEMPLATES = [ TEMPLATES = [
@@ -120,6 +129,21 @@ UNFOLD = {
"SITE_SUBHEADER": "Jobs, attempts, leases, and events", "SITE_SUBHEADER": "Jobs, attempts, leases, and events",
} }
INTERNAL_IPS = [
ip.strip()
for ip in os.getenv("DJANGO_INTERNAL_IPS", "127.0.0.1,localhost,host.docker.internal").split(",")
if ip.strip()
]
def show_debug_toolbar(_request):
return ENABLE_DJANGO_DEBUG_TOOLBAR
DEBUG_TOOLBAR_CONFIG = {
"SHOW_TOOLBAR_CALLBACK": show_debug_toolbar,
}
JOB_WORKER_THREADS = int(os.getenv("JOB_WORKER_THREADS", "4")) JOB_WORKER_THREADS = int(os.getenv("JOB_WORKER_THREADS", "4"))
JOB_WORKER_POLL_INTERVAL_MS = int(os.getenv("JOB_WORKER_POLL_INTERVAL_MS", "500")) JOB_WORKER_POLL_INTERVAL_MS = int(os.getenv("JOB_WORKER_POLL_INTERVAL_MS", "500"))
JOB_WORKER_LEASE_SECONDS = int(os.getenv("JOB_WORKER_LEASE_SECONDS", "30")) JOB_WORKER_LEASE_SECONDS = int(os.getenv("JOB_WORKER_LEASE_SECONDS", "30"))

View File

@@ -1,3 +1,4 @@
from django.conf import settings
from django.contrib import admin from django.contrib import admin
from django.urls import include, path from django.urls import include, path
from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
@@ -31,3 +32,6 @@ urlpatterns = [
path("api/config/", ConfigAPIView.as_view(), name="config"), path("api/config/", ConfigAPIView.as_view(), name="config"),
path("api/", include("jobs.urls")), path("api/", include("jobs.urls")),
] ]
if settings.ENABLE_DJANGO_DEBUG_TOOLBAR:
urlpatterns.append(path("__debug__/", include("debug_toolbar.urls")))

View File

@@ -9,3 +9,25 @@ select = ["E", "W", "F", "I", "B", "C4", "UP", "DJ", "SIM"]
"jobs/models.py" = ["DJ001"] "jobs/models.py" = ["DJ001"]
"jobs/migrations/*.py" = ["E501"] "jobs/migrations/*.py" = ["E501"]
"jobs/tests/*.py" = ["DJ001"] "jobs/tests/*.py" = ["DJ001"]
[tool.coverage.run]
branch = true
source = ["config", "jobs"]
omit = [
"config/asgi.py",
"config/wsgi.py",
"jobs/management/*",
"*/migrations/*",
"*/tests/*",
"manage.py",
]
[tool.coverage.report]
show_missing = true
skip_covered = true
fail_under = 60
exclude_lines = [
"pragma: no cover",
"if __name__ == .__main__.:",
"raise NotImplementedError",
]

View File

@@ -5,6 +5,8 @@ drf-spectacular>=0.28,<0.29
python-dotenv>=1.1,<2.0 python-dotenv>=1.1,<2.0
psycopg[binary]>=3.2,<4.0 psycopg[binary]>=3.2,<4.0
django-unfold>=0.76,<1.0 django-unfold>=0.76,<1.0
django-debug-toolbar>=5.0,<6.0
pytest>=8.0,<9.0 pytest>=8.0,<9.0
pytest-django>=4.9,<5.0 pytest-django>=4.9,<5.0
pytest-cov>=6.0,<7.0
ruff>=0.15,<0.16 ruff>=0.15,<0.16

View File

@@ -22,6 +22,7 @@ services:
command: sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000" command: sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
environment: environment:
DEBUG: "true" DEBUG: "true"
ENABLE_DJANGO_DEBUG_TOOLBAR: "true"
DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,0.0.0.0 DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,0.0.0.0
DJANGO_CORS_ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173 DJANGO_CORS_ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173
POSTGRES_DB: ${POSTGRES_DB:-job_queue} POSTGRES_DB: ${POSTGRES_DB:-job_queue}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.0 MiB

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 MiB

After

Width:  |  Height:  |  Size: 1.1 MiB

View File

@@ -55,18 +55,7 @@ const slides = [
] ]
}, },
{ {
eyebrow: '06 / Safe Claim', eyebrow: '06 / State',
title: 'PostgreSQL Chooses the Winner',
subtitle: 'Atomic claim uses row locks and SKIP LOCKED to avoid double execution.',
image: 'images/06-claim.png',
notes: [
'The worker selects the next eligible queued job ordered by priority, available_at, created_at, and id.',
'Inside one transaction, it uses <code>SELECT ... FOR UPDATE SKIP LOCKED</code>, then marks the row running, increments attempts, and sets the lease.',
'When many threads race, one locks the row; the others skip it instead of waiting and accidentally claiming the same job.'
]
},
{
eyebrow: '07 / State',
title: 'Status Transitions Are Explicit', title: 'Status Transitions Are Explicit',
subtitle: 'The queue allows only meaningful movement between queued, running, succeeded, and failed.', subtitle: 'The queue allows only meaningful movement between queued, running, succeeded, and failed.',
image: 'images/07-state.png', image: 'images/07-state.png',
@@ -76,6 +65,17 @@ const slides = [
'When attempts are exhausted, the job becomes failed. Manual retry can move failed back to queued and resets the attempt count.' 'When attempts are exhausted, the job becomes failed. Manual retry can move failed back to queued and resets the attempt count.'
] ]
}, },
{
eyebrow: '07 / Safe Claim',
title: 'PostgreSQL Chooses the Winner',
subtitle: 'Atomic claim uses row locks and SKIP LOCKED to avoid double execution.',
image: 'images/06-claim.png',
notes: [
'The worker selects the next eligible queued job ordered by priority, available_at, created_at, and id.',
'Inside one transaction, it uses <code>SELECT ... FOR UPDATE SKIP LOCKED</code>, then marks the row running, increments attempts, and sets the lease.',
'When many threads race, one locks the row; the others skip it instead of waiting and accidentally claiming the same job.'
]
},
{ {
eyebrow: '08 / Ownership', eyebrow: '08 / Ownership',
title: 'Current Owner + Attempt Is the Write Token', title: 'Current Owner + Attempt Is the Write Token',