Switching to django celery for workers

app_urls/core/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

app_urls/core/celery.py

@@ -0,0 +1,14 @@
+# core/celery.py
+import os
+from celery import Celery
+
+# Set default Django settings module
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+app = Celery('core')
+
+# Load config from Django settings, namespace CELERY
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Auto-discover tasks from all registered Django app configs
+app.autodiscover_tasks()

app_urls/core/settings.py

@@ -12,14 +12,12 @@ https://docs.djangoproject.com/en/5.1/ref/settings/
 
 from pathlib import Path
 import os
-from typing import Dict
-from scheduler.types import SchedulerConfiguration, Broker, QueueConfiguration
-
+# Queues and routing
+from kombu import Queue
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
-
 # Quick-start development settings - unsuitable for production
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -40,7 +38,7 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-    'scheduler',
+    'django_celery_beat',
     'fetcher',
 ]
 
@@ -110,27 +108,21 @@ CACHES = {
     }
 }
 
-SCHEDULER_CONFIG = SchedulerConfiguration(
-    EXECUTIONS_IN_PAGE=20,
-    SCHEDULER_INTERVAL=10,
-    BROKER=Broker.REDIS,
-    CALLBACK_TIMEOUT=60,  # Callback timeout in seconds (success/failure/stopped)
-    # Default values, can be overriden per task/job
-    DEFAULT_SUCCESS_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep successful job results
-    DEFAULT_FAILURE_TTL=365 * 24 * 60 * 60,  # Time To Live (TTL) in seconds to keep job failure information
-    DEFAULT_JOB_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep job information
-    DEFAULT_JOB_TIMEOUT=os.environ.get("JOB_DEFAULT_TIMEOUT", 60*30), # timeout (seconds) for a job
-    # General configuration values
-    DEFAULT_WORKER_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep worker information after last heartbeat
-    DEFAULT_MAINTENANCE_TASK_INTERVAL=10 * 60,  # The interval to run maintenance tasks in seconds. 10 minutes.
-    DEFAULT_JOB_MONITORING_INTERVAL=30,  # The interval to monitor jobs in seconds.
-    SCHEDULER_FALLBACK_PERIOD_SECS=120,  # Period (secs) to wait before requiring to reacquire locks
+
+
+# Celery configuration
+CELERY_BROKER_URL = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0))
+CELERY_RESULT_BACKEND = 'redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB_RESULTS", 1))
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+
+# Celery Beat scheduler (required for django-celery-beat to work)
+CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler'
+
+CELERY_TASK_QUEUES = (
+    Queue('default'),
+    Queue('low'),
 )
-SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = {
-    'default': QueueConfiguration(URL='redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0))),
-    'high': QueueConfiguration(URL='redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0))),
-    'low': QueueConfiguration(URL='redis://{}:{}/{}'.format(os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", 6379), os.environ.get("REDIS_DB", 0))),
-}
 
 
 # Password validation

app_urls/core/urls.py

@@ -19,6 +19,5 @@ from django.urls import path, include
 
 urlpatterns = [
     path('admin/', admin.site.urls),
-    path('scheduler/', include('scheduler.urls')),
     path('', include('fetcher.urls')),
 ]