From 60f966d49bbe0d82e292d491e0071d8724ae021a Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Sun, 6 Jul 2025 12:44:32 +0200 Subject: [PATCH] Implement analyst evaluation mode The idea behind this mode is that we are not really shadowing and trust the event feed results. However, we want to judge "interesting" runs locally to get useful information without the judging capacity to judge all testcases due to limited judgehost assignment. We do not consider 'TLE' or 'AC' interesting, as rerunning will not yield much more information. We consider 'WA' very interesting and prioritize the judging, but allow manual judging to overtake the priority. We consider 'CE' somewhat interesting, but downprioritize them a lot. For other verdicts, keep the normal priority. --- etc/db-config.yaml | 5 ++- .../Controller/API/JudgehostController.php | 4 +- webapp/src/Service/DOMJudgeService.php | 12 ++--- .../Service/ExternalContestSourceService.php | 44 ++++++++++++++++++- webapp/src/Service/SubmissionService.php | 9 +++- .../Integration/QueuetaskIntegrationTest.php | 1 + 6 files changed, 63 insertions(+), 12 deletions(-) diff --git a/etc/db-config.yaml b/etc/db-config.yaml index 665a48a6e8..a741b2a469 100644 --- a/etc/db-config.yaml +++ b/etc/db-config.yaml @@ -118,12 +118,13 @@ type: int default_value: 1 public: false - description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system. + description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases. options: 1: Lazy 2: Full judging 3: Only on request - regex: /^[123]$/ + 4: Analyst mode + regex: /^[1234]$/ error_message: A value between 1 and 3 is required. - name: judgehost_warning type: int diff --git a/webapp/src/Controller/API/JudgehostController.php b/webapp/src/Controller/API/JudgehostController.php index 5f9523ff4e..cca4a7c7df 100644 --- a/webapp/src/Controller/API/JudgehostController.php +++ b/webapp/src/Controller/API/JudgehostController.php @@ -1076,7 +1076,7 @@ private function addSingleJudgingRun( throw new BadMethodCallException('internal bug: the evaluated result changed during judging'); } - if ($lazyEval !== DOMJudgeService::EVAL_FULL) { + if ($lazyEval !== DOMJudgeService::EVAL_FULL && $lazyEval !== DOMJudgeService::EVAL_ANALYST) { // We don't want to continue on this problem, even if there's spare resources. $this->em->getConnection()->executeStatement( 'UPDATE judgetask SET valid=0, priority=:priority' @@ -1086,7 +1086,7 @@ private function addSingleJudgingRun( 'jobid' => $judgingRun->getJudgingid(), ] ); - } else { + } elseif ($lazyEval !== DOMJudgeService::EVAL_ANALYST) { // Decrease priority of remaining unassigned judging runs. $this->em->getConnection()->executeStatement( 'UPDATE judgetask SET priority=:priority' diff --git a/webapp/src/Service/DOMJudgeService.php b/webapp/src/Service/DOMJudgeService.php index 132beee23c..561951de8c 100644 --- a/webapp/src/Service/DOMJudgeService.php +++ b/webapp/src/Service/DOMJudgeService.php @@ -73,6 +73,7 @@ class DOMJudgeService final public const EVAL_LAZY = 1; final public const EVAL_FULL = 2; final public const EVAL_DEMAND = 3; + final public const EVAL_ANALYST = 4; // Regex external identifiers must adhere to. Note that we are not checking whether it // does not start with a dot or dash or ends with a dot. We could but it would make the @@ -1181,7 +1182,7 @@ public function unblockJudgeTasks(): void } } - public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void + public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); @@ -1194,7 +1195,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas return; } - $this->actuallyCreateJudgetasks($priority, $judging, $overshoot); + $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid); $team = $submission->getTeam(); $result = $this->em->createQueryBuilder() @@ -1212,7 +1213,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their // deserved and timely judgement. - // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent" + // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent" // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue // for other reasons, for example an internal error for a problem or language. @@ -1575,12 +1576,12 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan return !$evalOnDemand; } - private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void + private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); // We use a mass insert query, since that is way faster than doing a separate insert for each testcase. - // We first insert judgetasks, then select their ID's and finally insert the judging runs. + // We first insert judgetasks, then select their IDs and finally insert the judging runs. // Step 1: Create the template for the judgetasks. $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable(); @@ -1588,6 +1589,7 @@ private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $ ':type' => JudgeTaskType::JUDGING_RUN, ':submitid' => $submission->getSubmitid(), ':priority' => $priority, + ':valid' => $valid ? 1 : 0, ':jobid' => $judging->getJudgingid(), ':uuid' => $judging->getUuid(), ':compile_script_id' => $compileExecutable->getImmutableExecId(), diff --git a/webapp/src/Service/ExternalContestSourceService.php b/webapp/src/Service/ExternalContestSourceService.php index 719aee8e30..94198d0622 100644 --- a/webapp/src/Service/ExternalContestSourceService.php +++ b/webapp/src/Service/ExternalContestSourceService.php @@ -27,6 +27,7 @@ use App\Entity\ExternalJudgement; use App\Entity\ExternalRun; use App\Entity\ExternalSourceWarning; +use App\Entity\JudgeTask; use App\Entity\Language; use App\Entity\Problem; use App\Entity\Submission; @@ -1758,13 +1759,13 @@ protected function importRun(Event $event, EventData $data): void } // First, load the external run. + $persist = false; $run = $this->em ->getRepository(ExternalRun::class) ->findOneBy([ 'contest' => $this->getSourceContest(), 'externalid' => $runId, ]); - $persist = false; if (!$run) { $run = new ExternalRun(); $run @@ -1839,9 +1840,50 @@ protected function importRun(Event $event, EventData $data): void if ($persist) { $this->em->persist($run); } + + $lazyEval = $this->config->get('lazy_eval_results'); + if ($lazyEval === DOMJudgeService::EVAL_ANALYST) { + // Check if we want to judge this testcase locally to provide useful information for analysts + $priority = $this->getAnalystRunPriority($run); + if ($priority !== null) { + // Make the judgetask valid and assign running priority if no judgehost has picked it up yet. + $this->em->createQueryBuilder() + ->update(JudgeTask::class, 'jt') + ->set('jt.valid', true) + ->set('jt.priority', $priority) + ->andWhere('jt.testcase_id = :testcase_id') + ->andWhere('jt.submission = :submission') + ->andWhere('jt.judgehost IS NULL') + ->setParameter('testcase_id', $testcase->getTestcaseid()) + ->setParameter('submission', $externalJudgement->getSubmission()) + ->getQuery() + ->execute(); + } + } + $this->em->flush(); } + /** + * Checks if this run is interesting to judge locally for more analysis results. + * @param ExternalRun $run + * @return int The judging priority if it should be run locally, null otherwise. + */ + protected function getAnalystRunPriority(ExternalRun $run): int | null { + return match ($run->getResult()) { + // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time. + 'timelimit' => null, + // We often do not get new useful information for judging correct testcases. + 'correct' => null, + // Wrong answers are interesting for the analysts, assign a high priority but below manual judging. + 'wrong-answer' => -9, + // Compile errors could be interesting to see what went wrong, assign a low priority. + 'compiler-error' => 9, + // Otherwise, judge with normal priority. + default => 0, + }; + } + protected function processPendingEvents(string $type, string|int $id): void { // Process pending events. diff --git a/webapp/src/Service/SubmissionService.php b/webapp/src/Service/SubmissionService.php index 7dc699d43e..ec9f994549 100644 --- a/webapp/src/Service/SubmissionService.php +++ b/webapp/src/Service/SubmissionService.php @@ -726,8 +726,13 @@ public function submitSolution( // This is so that we can use the submitid/judgingid below. $this->em->flush(); - $this->dj->maybeCreateJudgeTasks($judging, - $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT); + $priority = match ($source) { + SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW, + default => JudgeTask::PRIORITY_DEFAULT, + }; + // Create judgetask as invalid when evaluating as analyst. + $lazyEval = $this->config->get('lazy_eval_results'); + $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: $lazyEval !== DOMJudgeService::EVAL_ANALYST); } $this->em->wrapInTransaction(function () use ($contest, $submission) { diff --git a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php index e39e47e3e0..1e5a7ded9c 100644 --- a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php +++ b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php @@ -60,6 +60,7 @@ protected function setUp(): void 'shadow_mode' => 0, 'sourcefiles_limit' => 1, 'sourcesize_limit' => 1024*256, + 'lazy_eval_results' => 1, ]; $this->config = $this->createMock(ConfigurationService::class);