diff --git a/.env_sample b/.env_sample index 9c00eafe7..3ee604941 100644 --- a/.env_sample +++ b/.env_sample @@ -116,3 +116,7 @@ ENABLE_SIGN_IN=True # ----------------------------------------------------------------------------- LOG_LEVEL=info SERIALIZED=false + + +# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature) +#DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS=2000 diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py index 69bc44229..b12a131d2 100644 --- a/src/apps/competitions/admin.py +++ b/src/apps/competitions/admin.py @@ -2,7 +2,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple from django.contrib.auth.models import Group - +from competitions.models import Submission from django.utils.translation import gettext_lazy as _ import json import csv @@ -166,6 +166,15 @@ def CompetitionExport_as_json(modeladmin, request, queryset): return HttpResponse(json.dumps(email_list), content_type="application/json") +# This will export the email of all the selected competition creators, removing duplications and banned users +@admin.display(description="Fail Submissions") +def Fail_submissions(modeladmin, request, queryset): + for obj in queryset: + obj.cancel(status=Submission.FAILED) + obj.status_details = "Failed Manually by Site Admin action" + obj.save() + + class QueueFilter(InputFilter): # Human-readable title which will be displayed in the # right admin sidebar just above the filter options. @@ -253,6 +262,19 @@ def queryset(self, request, queryset): return queryset.filter(phase__competition__created_by__username=value) +class StatusDetailsFilter(InputFilter): + # Human-readable title which will be displayed in the + # right admin sidebar just above the filter options. + title = _("Status Details") + # Parameter for the filter that will be used in the URL query. + parameter_name = "status_details" + + def queryset(self, request, queryset): + if self.value() is not None: + value = self.value() + return queryset.filter(status_details=value) + + class SubmissionExpansion(admin.ModelAdmin): # Raw Id Fields changes the field from displaying everything in a drop down menu into an id fields, which makes the page loads much faster (removes huge SELECT from the database) raw_id_fields = [ @@ -270,7 +292,7 @@ class SubmissionExpansion(admin.ModelAdmin): ] search_fields = ["id", "owner__username", "phase__competition__title", "task__name"] ordering = ('-id',) - actions = [SubmissionsExport_as_csv] + actions = [SubmissionsExport_as_csv, Fail_submissions] list_display = [ "id", "owner", @@ -287,6 +309,7 @@ class SubmissionExpansion(admin.ModelAdmin): "status", CompetitionOrganizerFilter, QueueFilter, + StatusDetailsFilter, ] fieldsets = [ ( diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 3d7d8abcb..de4028239 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -923,15 +923,86 @@ def update_phase_statuses(): @app.task(queue="site-worker") def submission_status_cleanup(): + # Recover submissions stuck in any non-terminal state + # There are multiple special cases that we have to deal with: + # - Submission stuck in Submitted, Preparing, Running or Scoring because of a network error or a problem on the compute worker (simple case this code aims to fix) + # - RabbitMQ failure (usually means submissions are stuck in Submitting) + # - Competitions with Auto Run-Submissions disabled (submissions appear "stuck" in Submitted until an organizer runs them manually) + # - Submission that appear to be stuck in Submitted, Preparing, Running, Scoring but are not for multiple reasons : + # - Lack of Compute workers in a queue, the submission is not stuck but waiting its turn + # - Submission file is heavy and/or the compute worker has slow network speeds + # - Combination of few compute workers + long submission execution time (allowed by a high execution time limit in the competition settings) + non_terminal_statuses = [ + Submission.SUBMITTING, + Submission.SUBMITTED, + Submission.PREPARING, + Submission.RUNNING, + Submission.SCORING, + ] submissions = Submission.objects.filter( - status=Submission.RUNNING, has_children=False - ).select_related("phase", "parent") + status__in=non_terminal_statuses, + has_children=False, + ).select_related('phase', 'parent') for sub in submissions: - if sub.started_when < now() - timedelta( - milliseconds=(3600000 * 24) + sub.phase.execution_time_limit - ): - if sub.parent is not None: - sub.parent.cancel(status=Submission.FAILED) + # Use started_when for Running submissions, created_when as fallback for others + # The deadline waits for 30 minutes after the phase execution time limit before failing submissions (making sure big submissions have time to upload) + if sub.started_when is not None: + reference_time = sub.started_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 30) + sub.phase.execution_time_limit + ) + + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + # The first if will filter out Preparing, Running and Scoring submissions. Only Submitting and Submitted are left to deal with + else: + # If a submission is stuck in SUBMITTING for 10 minutes after its creation + # then Rabbit might have failed, in which case the submission is unrecoverable + if sub.status == "SUBMITTING": + reference_time = sub.created_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 10) + ) + + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)" + sub_edit.save() else: - sub.cancel(status=Submission.FAILED) + # Special case where the competition has Auto Run Submission set to True. + # Otherwise, we might fail submissions if the organizer is not fast enough to start the submission + # TODO: Better logic to not Fail submissions that are stuck because of low Queue capacity + if sub.phase.competition.auto_run_submissions: + reference_time = sub.created_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 30) + sub.phase.execution_time_limit + ) + + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() diff --git a/src/apps/competitions/tests/test_submissions.py b/src/apps/competitions/tests/test_submissions.py index ee5cdc850..2429a05c2 100644 --- a/src/apps/competitions/tests/test_submissions.py +++ b/src/apps/competitions/tests/test_submissions.py @@ -427,6 +427,51 @@ def test_submissions_are_cancelled_if_running_24_hours_past_execution_time_limit assert self.submission_pass.status == Submission.RUNNING assert self.submission_fail.status == Submission.FAILED + def test_cleanup_recovers_stuck_submitted_submissions(self): + """Submissions stuck in Submitted should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.SUBMITTED + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_recovers_stuck_preparing_submissions(self): + """Submissions stuck in Preparing should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.PREPARING + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_recovers_stuck_scoring_submissions(self): + """Submissions stuck in Scoring should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.SCORING + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_does_not_touch_recent_non_terminal_submissions(self): + """Recent submissions in non-terminal states should NOT be cleaned up.""" + for status in [Submission.SUBMITTED, Submission.PREPARING, Submission.SCORING]: + sub = self.make_submission() + sub.status = status + sub.created_when = timezone.now() + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == status, f"Recent {status} submission should not be cleaned up" + def test_cancelling_parent_submission_cancels_all_children(self): self.parent_submission = self.make_submission() self.parent_submission.has_children = True diff --git a/src/settings/base.py b/src/settings/base.py index 3f39a2448..72bb195b8 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -257,7 +257,7 @@ }, 'submission_status_cleanup': { 'task': 'competitions.tasks.submission_status_cleanup', - 'schedule': timedelta(seconds=3600) + 'schedule': timedelta(seconds=7200) }, 'create_storage_analytics_snapshot': { 'task': 'analytics.tasks.create_storage_analytics_snapshot', @@ -584,3 +584,6 @@ def setup_celery_logging(**kwargs): # ============================================================================= ENABLE_SIGN_UP = os.environ.get('ENABLE_SIGN_UP', 'True').lower() == 'true' ENABLE_SIGN_IN = os.environ.get('ENABLE_SIGN_IN', 'True').lower() == 'true' + +# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature) +DATA_UPLOAD_MAX_NUMBER_FIELDS = int(os.environ.get('DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS', 2000))