diff --git a/etc/db-config.yaml b/etc/db-config.yaml index 665a48a6e8..a741b2a469 100644 --- a/etc/db-config.yaml +++ b/etc/db-config.yaml @@ -118,12 +118,13 @@ type: int default_value: 1 public: false - description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging and is typically used when running as analyst system. + description: Lazy evaluation of results? If enabled, stops judging as soon as a highest priority result is found, otherwise always all testcases will be judged. On request will not auto-start judging. Analyst mode tries to judge only interesting testcases. options: 1: Lazy 2: Full judging 3: Only on request - regex: /^[123]$/ + 4: Analyst mode + regex: /^[1234]$/ error_message: A value between 1 and 3 is required. - name: judgehost_warning type: int diff --git a/webapp/src/Controller/API/JudgehostController.php b/webapp/src/Controller/API/JudgehostController.php index 5f9523ff4e..cca4a7c7df 100644 --- a/webapp/src/Controller/API/JudgehostController.php +++ b/webapp/src/Controller/API/JudgehostController.php @@ -1076,7 +1076,7 @@ private function addSingleJudgingRun( throw new BadMethodCallException('internal bug: the evaluated result changed during judging'); } - if ($lazyEval !== DOMJudgeService::EVAL_FULL) { + if ($lazyEval !== DOMJudgeService::EVAL_FULL && $lazyEval !== DOMJudgeService::EVAL_ANALYST) { // We don't want to continue on this problem, even if there's spare resources. $this->em->getConnection()->executeStatement( 'UPDATE judgetask SET valid=0, priority=:priority' @@ -1086,7 +1086,7 @@ private function addSingleJudgingRun( 'jobid' => $judgingRun->getJudgingid(), ] ); - } else { + } elseif ($lazyEval !== DOMJudgeService::EVAL_ANALYST) { // Decrease priority of remaining unassigned judging runs. $this->em->getConnection()->executeStatement( 'UPDATE judgetask SET priority=:priority' diff --git a/webapp/src/Service/DOMJudgeService.php b/webapp/src/Service/DOMJudgeService.php index 132beee23c..561951de8c 100644 --- a/webapp/src/Service/DOMJudgeService.php +++ b/webapp/src/Service/DOMJudgeService.php @@ -73,6 +73,7 @@ class DOMJudgeService final public const EVAL_LAZY = 1; final public const EVAL_FULL = 2; final public const EVAL_DEMAND = 3; + final public const EVAL_ANALYST = 4; // Regex external identifiers must adhere to. Note that we are not checking whether it // does not start with a dot or dash or ends with a dot. We could but it would make the @@ -1181,7 +1182,7 @@ public function unblockJudgeTasks(): void } } - public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0): void + public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTask::PRIORITY_DEFAULT, bool $manualRequest = false, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); @@ -1194,7 +1195,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas return; } - $this->actuallyCreateJudgetasks($priority, $judging, $overshoot); + $this->actuallyCreateJudgetasks($priority, $judging, $overshoot, $valid); $team = $submission->getTeam(); $result = $this->em->createQueryBuilder() @@ -1212,7 +1213,7 @@ public function maybeCreateJudgeTasks(Judging $judging, int $priority = JudgeTas // Teams that submit frequently slow down the judge queue but should not be able to starve other teams of their // deserved and timely judgement. - // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definiition of "recent" + // For every "recent" pending job in the queue by that team, add a penalty (60s). Our definition of "recent" // includes all submissions that have been placed at a virtual time (including penalty) more recent than 60s // ago. This is done in order to avoid punishing teams who submit while their submissions are stuck in the queue // for other reasons, for example an internal error for a problem or language. @@ -1575,12 +1576,12 @@ private function allowJudge(ContestProblem $problem, Submission $submission, Lan return !$evalOnDemand; } - private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0): void + private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $overshoot = 0, bool $valid = true): void { $submission = $judging->getSubmission(); $problem = $submission->getContestProblem(); // We use a mass insert query, since that is way faster than doing a separate insert for each testcase. - // We first insert judgetasks, then select their ID's and finally insert the judging runs. + // We first insert judgetasks, then select their IDs and finally insert the judging runs. // Step 1: Create the template for the judgetasks. $compileExecutable = $submission->getLanguage()->getCompileExecutable()->getImmutableExecutable(); @@ -1588,6 +1589,7 @@ private function actuallyCreateJudgetasks(int $priority, Judging $judging, int $ ':type' => JudgeTaskType::JUDGING_RUN, ':submitid' => $submission->getSubmitid(), ':priority' => $priority, + ':valid' => $valid ? 1 : 0, ':jobid' => $judging->getJudgingid(), ':uuid' => $judging->getUuid(), ':compile_script_id' => $compileExecutable->getImmutableExecId(), diff --git a/webapp/src/Service/ExternalContestSourceService.php b/webapp/src/Service/ExternalContestSourceService.php index 719aee8e30..94198d0622 100644 --- a/webapp/src/Service/ExternalContestSourceService.php +++ b/webapp/src/Service/ExternalContestSourceService.php @@ -27,6 +27,7 @@ use App\Entity\ExternalJudgement; use App\Entity\ExternalRun; use App\Entity\ExternalSourceWarning; +use App\Entity\JudgeTask; use App\Entity\Language; use App\Entity\Problem; use App\Entity\Submission; @@ -1758,13 +1759,13 @@ protected function importRun(Event $event, EventData $data): void } // First, load the external run. + $persist = false; $run = $this->em ->getRepository(ExternalRun::class) ->findOneBy([ 'contest' => $this->getSourceContest(), 'externalid' => $runId, ]); - $persist = false; if (!$run) { $run = new ExternalRun(); $run @@ -1839,9 +1840,50 @@ protected function importRun(Event $event, EventData $data): void if ($persist) { $this->em->persist($run); } + + $lazyEval = $this->config->get('lazy_eval_results'); + if ($lazyEval === DOMJudgeService::EVAL_ANALYST) { + // Check if we want to judge this testcase locally to provide useful information for analysts + $priority = $this->getAnalystRunPriority($run); + if ($priority !== null) { + // Make the judgetask valid and assign running priority if no judgehost has picked it up yet. + $this->em->createQueryBuilder() + ->update(JudgeTask::class, 'jt') + ->set('jt.valid', true) + ->set('jt.priority', $priority) + ->andWhere('jt.testcase_id = :testcase_id') + ->andWhere('jt.submission = :submission') + ->andWhere('jt.judgehost IS NULL') + ->setParameter('testcase_id', $testcase->getTestcaseid()) + ->setParameter('submission', $externalJudgement->getSubmission()) + ->getQuery() + ->execute(); + } + } + $this->em->flush(); } + /** + * Checks if this run is interesting to judge locally for more analysis results. + * @param ExternalRun $run + * @return int The judging priority if it should be run locally, null otherwise. + */ + protected function getAnalystRunPriority(ExternalRun $run): int | null { + return match ($run->getResult()) { + // We will not get any new useful information for TLE testcases, while they take a lot of judgedaemon time. + 'timelimit' => null, + // We often do not get new useful information for judging correct testcases. + 'correct' => null, + // Wrong answers are interesting for the analysts, assign a high priority but below manual judging. + 'wrong-answer' => -9, + // Compile errors could be interesting to see what went wrong, assign a low priority. + 'compiler-error' => 9, + // Otherwise, judge with normal priority. + default => 0, + }; + } + protected function processPendingEvents(string $type, string|int $id): void { // Process pending events. diff --git a/webapp/src/Service/SubmissionService.php b/webapp/src/Service/SubmissionService.php index 7dc699d43e..ec9f994549 100644 --- a/webapp/src/Service/SubmissionService.php +++ b/webapp/src/Service/SubmissionService.php @@ -726,8 +726,13 @@ public function submitSolution( // This is so that we can use the submitid/judgingid below. $this->em->flush(); - $this->dj->maybeCreateJudgeTasks($judging, - $source === SubmissionSource::PROBLEM_IMPORT ? JudgeTask::PRIORITY_LOW : JudgeTask::PRIORITY_DEFAULT); + $priority = match ($source) { + SubmissionSource::PROBLEM_IMPORT => JudgeTask::PRIORITY_LOW, + default => JudgeTask::PRIORITY_DEFAULT, + }; + // Create judgetask as invalid when evaluating as analyst. + $lazyEval = $this->config->get('lazy_eval_results'); + $this->dj->maybeCreateJudgeTasks($judging, $priority, valid: $lazyEval !== DOMJudgeService::EVAL_ANALYST); } $this->em->wrapInTransaction(function () use ($contest, $submission) { diff --git a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php index e39e47e3e0..1e5a7ded9c 100644 --- a/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php +++ b/webapp/tests/Unit/Integration/QueuetaskIntegrationTest.php @@ -60,6 +60,7 @@ protected function setUp(): void 'shadow_mode' => 0, 'sourcefiles_limit' => 1, 'sourcesize_limit' => 1024*256, + 'lazy_eval_results' => 1, ]; $this->config = $this->createMock(ConfigurationService::class);