feat: add pipeline observability and stats

src/core/worker/main.py

@@ -3,7 +3,9 @@ import signal
 
 from core.config import get_settings
 from core.worker.consumer import ProcessingWorker
-
+from core.worker.metrics import WorkerMetrics
+from core.worker.observability import WorkerObservabilityServer
+from core.worker.stats import WorkerRuntimeStats
 
 logging.basicConfig(
     level=logging.INFO,
@@ -19,8 +21,20 @@ logging.basicConfig(
 def main() -> None:
     settings = get_settings()
 
+    metrics = WorkerMetrics()
+
+    stats = WorkerRuntimeStats()
+
     worker = ProcessingWorker(
         settings=settings,
+        metrics=metrics,
+        stats=stats,
+    )
+
+    observability = WorkerObservabilityServer(
+        settings=settings,
+        metrics=metrics,
+        stats=stats,
     )
 
     def handle_shutdown(
@@ -44,7 +58,13 @@ def main() -> None:
         handle_shutdown,
     )
 
-    worker.run()
+    observability.start()
+
+    try:
+        worker.run()
+
+    finally:
+        observability.stop()
 
 
 if __name__ == "__main__":