Implement analyst evaluation mode

etc/db-config.yaml

@@ -118,12 +118,13 @@
             type: int
             default_value: 1
             public: false
-            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system.
+            description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases.
             options:
                 1: Lazy
                 2: Full judging
                 3: Only on request
-            regex: /^[123]$/
+                4: Analyst mode
+            regex: /^[1234]$/
             error_message: A value between 1 and 3 is required.
         -   name: judgehost_warning
             type: int

webapp/src/Controller/API/JudgehostController.php

@@ -1076,7 +1076,7 @@ private function addSingleJudgingRun(
                     throw new BadMethodCallException('internal bug: the evaluated result changed during judging');
                 }
 
-                if ($lazyEval !== DOMJudgeService::EVAL_FULL) {
+                if ($lazyEval !== DOMJudgeService::EVAL_FULL && $lazyEval !== DOMJudgeService::EVAL_ANALYST) {
                     // We don't want to continue on this problem, even if there's spare resources.
                     $this->em->getConnection()->executeStatement(
                         'UPDATE judgetask SET valid=0, priority=:priority'
@@ -1086,7 +1086,7 @@ private function addSingleJudgingRun(
                             'jobid' => $judgingRun->getJudgingid(),
                         ]
                     );
-                } else {
+                } elseif ($lazyEval !== DOMJudgeService::EVAL_ANALYST) {
                     // Decrease priority of remaining unassigned judging runs.
                     $this->em->getConnection()->executeStatement(
                         'UPDATE judgetask SET priority=:priority'

webapp/src/Service/DOMJudgeService.php

@@ -73,6 +73,7 @@ class DOMJudgeService
     final public const EVAL_LAZY = 1;
     final public const EVAL_FULL = 2;
     final public const EVAL_DEMAND = 3;
+    final public const EVAL_ANALYST = 4;
 
     // Regex external identifiers must adhere to. Note that we are not checking whether it
     // does not start with a dot or dash or ends with a dot. We could but it would make the
@@ -1181,7 +1182,7 @@ public function unblockJudgeTasks(): void
         }
     }
 
-    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void
+    public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
@@ -1194,7 +1195,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
             return;
         }
 
-        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot);
+        $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid);
 
         $team = $submission->getTeam();
         $result = $this->em->createQueryBuilder()
@@ -1212,7 +1213,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas
 
         // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their
         // deserved and timely judgement.
-        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent"
+        // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent"
         // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s
         // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue
         // for other reasons, for example an internal error for a problem or language.
@@ -1575,19 +1576,20 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan
         return !$evalOnDemand;
     }
 
-    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void
+    private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void
     {
         $submission = $judging->getSubmission();
         $problem    = $submission->getContestProblem();
         // We use a mass insert query, since that is way faster than doing a separate insert for each testcase.
-        // We first insert judgetasks, then select their ID's and finally insert the judging runs.
+        // We first insert judgetasks, then select their IDs and finally insert the judging runs.
 
         // Step 1: Create the template for the judgetasks.
         $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable();
         $judgetaskInsertParams = [
             ':type' => JudgeTaskType::JUDGING_RUN,
             ':submitid' => $submission->getSubmitid(),
             ':priority' => $priority,
+            ':valid' => $valid ? 1 : 0,
             ':jobid' => $judging->getJudgingid(),
             ':uuid' => $judging->getUuid(),
             ':compile_script_id' => $compileExecutable->getImmutableExecId(),

webapp/src/Service/ExternalContestSourceService.php

@@ -27,6 +27,7 @@
 use App\Entity\ExternalJudgement;
 use App\Entity\ExternalRun;
 use App\Entity\ExternalSourceWarning;
+use App\Entity\JudgeTask;
 use App\Entity\Language;
 use App\Entity\Problem;
 use App\Entity\Submission;
@@ -1758,13 +1759,13 @@ protected function importRun(Event $event, EventData $data): void
         }
 
         // First, load the external run.
+        $persist = false;
         $run     = $this->em
             ->getRepository(ExternalRun::class)
             ->findOneBy([
                             'contest'    => $this->getSourceContest(),
                             'externalid' => $runId,
                         ]);
-        $persist = false;
         if (!$run) {
             $run = new ExternalRun();
             $run
@@ -1839,9 +1840,50 @@ protected function importRun(Event $event, EventData $data): void
         if ($persist) {
             $this->em->persist($run);
         }
+
+        $lazyEval = $this->config->get('lazy_eval_results');
+        if ($lazyEval === DOMJudgeService::EVAL_ANALYST) {
+            // Check if we want to judge this testcase locally to provide useful information for analysts
+            $priority = $this->getAnalystRunPriority($run);
+            if ($priority !== null) {
+                // Make the judgetask valid and assign running priority if no judgehost has picked it up yet.
+                $this->em->createQueryBuilder()
+                        ->update(JudgeTask::class, 'jt')
+                        ->set('jt.valid', true)
+                        ->set('jt.priority', $priority)
+                        ->andWhere('jt.testcase_id = :testcase_id')
+                        ->andWhere('jt.submission = :submission')
+                        ->andWhere('jt.judgehost IS NULL')
+                        ->setParameter('testcase_id', $testcase->getTestcaseid())
+                        ->setParameter('submission', $externalJudgement->getSubmission())
+                        ->getQuery()
+                        ->execute();
+            }
+        }
+
         $this->em->flush();
     }
 
+    /**
+     * Checks if this run is interesting to judge locally for more analysis results.
+     * @param ExternalRun $run
+     * @return int The judging priority if it should be run locally, null otherwise.
+     */
+    protected function getAnalystRunPriority(ExternalRun $run): int | null {
+        return match ($run->getResult()) {
+            // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time.
+            'timelimit' => null,
+            // We often do not get new useful information for judging correct testcases.
+            'correct' => null,
+            // Wrong answers are interesting for the analysts, assign a high priority but below manual judging.
+            'wrong-answer' => -9,
+            // Compile errors could be interesting to see what went wrong, assign a low priority.
+            'compiler-error' => 9,
+            // Otherwise, judge with normal priority.
+            default => 0,
+        };
+    }
+
     protected function processPendingEvents(string $type, string|int $id): void
     {
         // Process pending events.

webapp/src/Service/SubmissionService.php

@@ -726,8 +726,13 @@ public function submitSolution(
             // This is so that we can use the submitid/judgingid below.
             $this->em->flush();
 
-            $this->dj->maybeCreateJudgeTasks($judging,
-                $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT);
+            $priority = match ($source) {
+                SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW,
+                default => JudgeTask::PRIORITY_DEFAULT,
+            };
+            // Create judgetask as invalid when evaluating as analyst.
+            $lazyEval = $this->config->get('lazy_eval_results');
+            $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: $lazyEval !== DOMJudgeService::EVAL_ANALYST);
         }
 
         $this->em->wrapInTransaction(function () use ($contest, $submission) {

webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php

@@ -60,6 +60,7 @@ protected function setUp(): void
             'shadow_mode' => 0,
             'sourcefiles_limit' => 1,
             'sourcesize_limit' => 1024*256,
+            'lazy_eval_results' => 1,
         ];
 
         $this->config = $this->createMock(ConfigurationService::class);