From 5a6b42fe84a6babddcdad99ba9610623e469f8fa Mon Sep 17 00:00:00 2001 From: AybH26 <58746253+AybH26@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:47:40 +0200 Subject: [PATCH 1/6] fix(scoring): re-enqueue scoring after commit to avoid stuck SCORING rows When the compute worker PATCHes a submission to status=SCORING, the API serializer used to call run_submission() synchronously inside the same DB transaction. If the broker (RabbitMQ) was unreachable at that exact moment, the status row would commit but the scoring task would never be published, leaving the submission stuck in SCORING forever (no recovery: the 24h cleanup only rescues RUNNING rows). Move the enqueue into transaction.on_commit so the task is only published after the SCORING status is durably committed, and explicitly mark the submission as Failed (with a clear status_details) if the publish still fails, so the row never stays in a non-terminal limbo state. Wrap update() in @transaction.atomic to make the commit boundary explicit. --- src/apps/api/serializers/submissions.py | 33 ++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 9c91737ca..769ef0131 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -1,11 +1,15 @@ import asyncio +import logging from os.path import basename from django.core.cache import cache from django.core.exceptions import ValidationError +from django.db import transaction from rest_framework import serializers from rest_framework.exceptions import PermissionDenied +logger = logging.getLogger(__name__) + from api.mixins import DefaultUserCreateMixin from api.serializers import leaderboards from api.serializers.tasks import TaskSerializer @@ -163,6 +167,7 @@ def validate(self, attrs): return data + @transaction.atomic def update(self, submission, validated_data): # Cannot change submission if secret key is not valid @@ -198,13 +203,29 @@ def update(self, submission, validated_data): self.instance.parent.save() if validated_data.get("status") == Submission.SCORING: - # Start scoring because we're "SCORING" status now from compute worker + # Re-enqueue scoring AFTER the new status is committed: otherwise the + # site-worker may pick the task up before the row reflects SCORING, + # and a broker error here would leave the row stuck in SCORING forever. from competitions.tasks import run_submission - # task = validated_data.get('task_pk') - # if not task: - # raise ValidationError('Cannot update submission. Task pk was not provided') - # task = Task.objects.get(id=task) - run_submission(submission.pk, tasks=[submission.task], is_scoring=True) + submission_pk = submission.pk + scoring_task = submission.task + + def _enqueue_scoring(): + try: + run_submission(submission_pk, tasks=[scoring_task], is_scoring=True) + except Exception: + logger.exception( + "Failed to re-enqueue scoring for submission %s; marking Failed", + submission_pk, + ) + Submission.objects.filter( + pk=submission_pk, status=Submission.SCORING, + ).update( + status=Submission.FAILED, + status_details="Broker unavailable when re-enqueuing scoring task", + ) + + transaction.on_commit(_enqueue_scoring) elif validated_data.get("status") == Submission.FINISHED: # We finished submission, no longer need to store submission stuff in Redis, free it up! From 15baf6d6e42ab48da82f7c2a0dd7a73349563006 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 26 Jun 2026 14:10:44 +0200 Subject: [PATCH 2/6] flake8 fixes --- src/apps/api/serializers/submissions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 769ef0131..11ac4578d 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -17,9 +17,10 @@ from competitions.models import Submission, SubmissionDetails, CompetitionParticipant, Phase from datasets.models import Data from utils.data import make_url_sassy - +from channels.layers import get_channel_layer from tasks.models import Task from queues.models import Queue +from competitions.tasks import run_submission class SubmissionSerializer(serializers.ModelSerializer): @@ -181,7 +182,7 @@ def update(self, submission, validated_data): # Update status if it is there in validated data if "status" in validated_data: # Received a status update, let the frontend know - from channels.layers import get_channel_layer + channel_layer = get_channel_layer() try: @@ -206,7 +207,7 @@ def update(self, submission, validated_data): # Re-enqueue scoring AFTER the new status is committed: otherwise the # site-worker may pick the task up before the row reflects SCORING, # and a broker error here would leave the row stuck in SCORING forever. - from competitions.tasks import run_submission + submission_pk = submission.pk scoring_task = submission.task From ea0a87022692363fe41334b0dcda653f49186907 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 26 Jun 2026 14:15:58 +0200 Subject: [PATCH 3/6] flake8 fixes --- src/apps/api/serializers/submissions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 11ac4578d..33b085a1f 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -8,8 +8,6 @@ from rest_framework import serializers from rest_framework.exceptions import PermissionDenied -logger = logging.getLogger(__name__) - from api.mixins import DefaultUserCreateMixin from api.serializers import leaderboards from api.serializers.tasks import TaskSerializer @@ -22,6 +20,8 @@ from queues.models import Queue from competitions.tasks import run_submission +logger = logging.getLogger(__name__) + class SubmissionSerializer(serializers.ModelSerializer): scores = SubmissionScoreSerializer(many=True) @@ -182,7 +182,6 @@ def update(self, submission, validated_data): # Update status if it is there in validated data if "status" in validated_data: # Received a status update, let the frontend know - channel_layer = get_channel_layer() try: @@ -207,7 +206,6 @@ def update(self, submission, validated_data): # Re-enqueue scoring AFTER the new status is committed: otherwise the # site-worker may pick the task up before the row reflects SCORING, # and a broker error here would leave the row stuck in SCORING forever. - submission_pk = submission.pk scoring_task = submission.task From 8a9b38a6dd98ca8e0009375a0920ac04b5e903b6 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 26 Jun 2026 14:22:07 +0200 Subject: [PATCH 4/6] Revert "flake8 fixes" This reverts commit d38a0131f5252286908d3af5d37936130bf06a4b. --- src/apps/api/serializers/submissions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 33b085a1f..11ac4578d 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -8,6 +8,8 @@ from rest_framework import serializers from rest_framework.exceptions import PermissionDenied +logger = logging.getLogger(__name__) + from api.mixins import DefaultUserCreateMixin from api.serializers import leaderboards from api.serializers.tasks import TaskSerializer @@ -20,8 +22,6 @@ from queues.models import Queue from competitions.tasks import run_submission -logger = logging.getLogger(__name__) - class SubmissionSerializer(serializers.ModelSerializer): scores = SubmissionScoreSerializer(many=True) @@ -182,6 +182,7 @@ def update(self, submission, validated_data): # Update status if it is there in validated data if "status" in validated_data: # Received a status update, let the frontend know + channel_layer = get_channel_layer() try: @@ -206,6 +207,7 @@ def update(self, submission, validated_data): # Re-enqueue scoring AFTER the new status is committed: otherwise the # site-worker may pick the task up before the row reflects SCORING, # and a broker error here would leave the row stuck in SCORING forever. + submission_pk = submission.pk scoring_task = submission.task From 38ce5f4ae1c193fea8f7ca4797f7d23efeb579cb Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 26 Jun 2026 14:22:26 +0200 Subject: [PATCH 5/6] Revert "flake8 fixes" This reverts commit 2aad1c99ea7e515092b0633c3e74c7b8d68c7405. --- src/apps/api/serializers/submissions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 11ac4578d..769ef0131 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -17,10 +17,9 @@ from competitions.models import Submission, SubmissionDetails, CompetitionParticipant, Phase from datasets.models import Data from utils.data import make_url_sassy -from channels.layers import get_channel_layer + from tasks.models import Task from queues.models import Queue -from competitions.tasks import run_submission class SubmissionSerializer(serializers.ModelSerializer): @@ -182,7 +181,7 @@ def update(self, submission, validated_data): # Update status if it is there in validated data if "status" in validated_data: # Received a status update, let the frontend know - + from channels.layers import get_channel_layer channel_layer = get_channel_layer() try: @@ -207,7 +206,7 @@ def update(self, submission, validated_data): # Re-enqueue scoring AFTER the new status is committed: otherwise the # site-worker may pick the task up before the row reflects SCORING, # and a broker error here would leave the row stuck in SCORING forever. - + from competitions.tasks import run_submission submission_pk = submission.pk scoring_task = submission.task From 25b78b5fcf5c8d43a26d034e0bb1ebdb2f481fda Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 26 Jun 2026 14:23:38 +0200 Subject: [PATCH 6/6] flake8 fixes --- src/apps/api/serializers/submissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index 769ef0131..db7ee886a 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -8,8 +8,6 @@ from rest_framework import serializers from rest_framework.exceptions import PermissionDenied -logger = logging.getLogger(__name__) - from api.mixins import DefaultUserCreateMixin from api.serializers import leaderboards from api.serializers.tasks import TaskSerializer @@ -21,6 +19,8 @@ from tasks.models import Task from queues.models import Queue +logger = logging.getLogger(__name__) + class SubmissionSerializer(serializers.ModelSerializer): scores = SubmissionScoreSerializer(many=True)