From 8350a2f14262f44b9191430689ac97003b5b9761 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 6 Jul 2025 15:49:53 +0200 Subject: [PATCH 01/12] Refactor internalApiRequest() and getEventFeedAction() for streaming via CLI Fetching the event feed from the API via `internalApiRequest` wasn't working since it returns a StreamedResponse. Fix this by only using an output buffer in `internalApiRequest` and only calling `ob_flush` from `getEventFeedAction` if the top-level output buffer allows it. Also return contents from `internalApiRequest` without JSON decoding it, since the event feed endpoint is NDJSON, *not* JSON. Explicitly decode the JSON at the other places calling internalApiRequest`. --- webapp/src/Command/CallApiActionCommand.php | 4 ++-- webapp/src/Controller/API/ContestController.php | 4 ++-- webapp/src/Service/DOMJudgeService.php | 12 +++++++----- webapp/src/Service/EventLogService.php | 15 ++++++++++----- webapp/src/Utils/Utils.php | 12 ++++++++++++ 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/webapp/src/Command/CallApiActionCommand.php b/webapp/src/Command/CallApiActionCommand.php index 47d98c2671..64a0fda581 100644 --- a/webapp/src/Command/CallApiActionCommand.php +++ b/webapp/src/Command/CallApiActionCommand.php @@ -144,7 +144,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } try { - $response = null; + $response = ''; $this->dj->withAllRoles(function () use ($input, $data, $files, &$response) { $response = $this->dj->internalApiRequest('/' . $input->getArgument('endpoint'), $input->getOption('method'), $data, $files, true); }, $user); @@ -154,7 +154,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $output->writeln(json_encode($response, JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)); + $output->writeln($response); return Command::SUCCESS; } } diff --git a/webapp/src/Controller/API/ContestController.php b/webapp/src/Controller/API/ContestController.php index 8565c31589..f6816e6938 100644 --- a/webapp/src/Controller/API/ContestController.php +++ b/webapp/src/Controller/API/ContestController.php @@ -871,7 +871,7 @@ public function getEventFeedAction( } echo Utils::jsonEncode($result) . "\n"; - ob_flush(); + Utils::ob_flush_if_possible(); flush(); $lastUpdate = Utils::now(); $lastIdSent = $event->getEventid(); @@ -896,7 +896,7 @@ public function getEventFeedAction( # Send keep alive every 10s. Guarantee according to spec is 120s. # However, nginx drops the connection if we don't update for 60s. echo "\n"; - ob_flush(); + Utils::ob_flush_if_possible(); flush(); $lastUpdate = $now; } diff --git a/webapp/src/Service/DOMJudgeService.php b/webapp/src/Service/DOMJudgeService.php index 8571dfa81f..5e8a92ca40 100644 --- a/webapp/src/Service/DOMJudgeService.php +++ b/webapp/src/Service/DOMJudgeService.php @@ -652,13 +652,15 @@ public function internalApiRequest(string $url, string $method = Request::METHOD return null; } - $content = $response->getContent(); - - if ($content === '') { - return null; + if ($response instanceof StreamedResponse) { + ob_start(flags: PHP_OUTPUT_HANDLER_REMOVABLE); + $response->sendContent(); + $content = ob_get_clean(); + } else { + $content = $response->getContent(); } - return Utils::jsonDecode($content); + return $content; } public function getDomjudgeEtcDir(): string diff --git a/webapp/src/Service/EventLogService.php b/webapp/src/Service/EventLogService.php index e400701c01..c31ad90a10 100644 --- a/webapp/src/Service/EventLogService.php +++ b/webapp/src/Service/EventLogService.php @@ -316,12 +316,12 @@ public function log( } $this->dj->withAllRoles(function () use ($query, $url, &$json) { - $json = $this->dj->internalApiRequest($url, Request::METHOD_GET, $query); + $response = $this->dj->internalApiRequest($url, Request::METHOD_GET, $query); }); - if ($json === null) { + if ($response === '') { $this->logger->warning( - "EventLogService::log got no JSON data from '%s'", [ $url ] + "EventLogService::log got no data from '%s'", [ $url ] ); // If we didn't get data from the API, then that is probably // because this particular data is not visible, for example @@ -329,6 +329,9 @@ public function log( // have data, there's also no point in trying to insert // anything in the eventlog table. return; + $json = null; + } else { + $json = Utils::jsonDecode($response); } } @@ -484,7 +487,8 @@ public function addMissingStateEvents(Contest $contest): void $url = sprintf('/contests/%s/awards', $contest->getExternalid()); $awards = []; $this->dj->withAllRoles(function () use ($url, &$awards) { - $awards = $this->dj->internalApiRequest($url); + $response = $this->dj->internalApiRequest($url); + if ( !empty($response) ) $awards = Utils::jsonDecode($response); }); foreach ($awards as $award) { $this->insertEvent($contest, 'awards', $award['id'], $award); @@ -625,7 +629,8 @@ public function initStaticEvents(Contest $contest): void $urlPart = $endpoint === 'contests' ? '' : ('/' . $endpoint); $url = sprintf('/contests/%s%s', $contestId, $urlPart); $this->dj->withAllRoles(function () use ($url, &$data) { - $data = $this->dj->internalApiRequest($url); + $response = $this->dj->internalApiRequest($url); + $data = (empty($response) ? null : Utils::jsonDecode($response)); }); // Get a partial reference to the contest, diff --git a/webapp/src/Utils/Utils.php b/webapp/src/Utils/Utils.php index 7d0a191a02..3ee9392380 100644 --- a/webapp/src/Utils/Utils.php +++ b/webapp/src/Utils/Utils.php @@ -1023,4 +1023,16 @@ public static function extendMaxExecutionTime(int $minimumMaxExecutionTime): voi ini_set('max_execution_time', $minimumMaxExecutionTime); } } + + /** + * Call ob_flush() unless the top-level output buffer does not allow it. + */ + public static function ob_flush_if_possible(): bool + { + $status = ob_get_status(); + if ( empty($status) || ($status['flags'] & PHP_OUTPUT_HANDLER_CLEANABLE) ) { + return ob_flush(); + } + return false; + } } From ee81fdf40267c31b61c633a64a0c5da2acac100e Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 6 Jul 2025 16:00:07 +0200 Subject: [PATCH 02/12] Add option to not JSON decode API responses This will be used by the contest export script for downloading the event feed (NDJSON) and other files. Also improve error reporting when an internal API call fails. --- misc-tools/dj_utils.py | 48 ++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/misc-tools/dj_utils.py b/misc-tools/dj_utils.py index dd27b9865f..ab44b923de 100644 --- a/misc-tools/dj_utils.py +++ b/misc-tools/dj_utils.py @@ -30,7 +30,7 @@ def confirm(message: str, default: bool) -> bool: return answer == 'y' -def parse_api_response(name: str, response: requests.Response): +def parse_api_response(name: str, response: requests.Response) -> bytes: # The connection worked, but we may have received an HTTP error if response.status_code >= 300: print(response.text) @@ -44,17 +44,10 @@ def parse_api_response(name: str, response: requests.Response): if response.status_code == 204: return None - # We got a successful HTTP response. It worked. Return the full response - try: - result = json.loads(response.text) - except json.decoder.JSONDecodeError: - print(response.text) - raise RuntimeError(f'Failed to JSON decode the response for API request {name}') - - return result + return response.content -def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): +def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this @@ -64,16 +57,18 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): name (str): the endpoint to call method (str): the method to use, GET or PUT are supported jsonData (dict): the JSON data to PUT. Only used when method is PUT + decode (bool): whether to decode the returned JSON data, default true Returns: - The endpoint contents. + The endpoint contents, either as raw bytes or JSON decoded. Raises: - RuntimeError when the response is not JSON or the HTTP status code is non 2xx. + RuntimeError when the HTTP status code is non-2xx or the response + cannot be JSON decoded. ''' if os.path.isdir(domjudge_webapp_folder_or_api_url): - return api_via_cli(name, method, {}, {}, jsonData) + result = api_via_cli(name, method, {}, {}, jsonData) else: global ca_check url = f'{domjudge_webapp_folder_or_api_url}/{name}' @@ -86,7 +81,7 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): if method == 'GET': response = requests.get(url, headers=headers, verify=ca_check, auth=auth) elif method == 'PUT': - response = requests.put(url, headers=headers, verify=ca_check, json=jsonData, auth=auth) + response = requests.put(url, headers=headers, verify=ca_check, auth=auth, json=jsonData) else: raise RuntimeError("Method not supported") except requests.exceptions.SSLError: @@ -99,7 +94,16 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}): return do_api_request(name) except requests.exceptions.RequestException as e: raise RuntimeError(e) - return parse_api_response(name, response) + result = parse_api_response(name, response) + + if decode: + try: + result = json.loads(result) + except json.decoder.JSONDecodeError as e: + print(result) + raise RuntimeError(f'Failed to JSON decode the response for API request {name}') + + return result def upload_file(name: str, apifilename: str, file: str, data: dict = {}): @@ -121,7 +125,7 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}): ''' if os.path.isdir(domjudge_webapp_folder_or_api_url): - return api_via_cli(name, 'POST', data, {apifilename: file}) + response = api_via_cli(name, 'POST', data, {apifilename: file}) else: global ca_check files = [(apifilename, open(file, 'rb'))] @@ -155,7 +159,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { jsonData (dict): the JSON data to use. Only used when method is POST or PUT Returns: - The parsed endpoint contents. + The raw endpoint contents. Raises: RuntimeError when the command exit code is not 0. @@ -180,10 +184,14 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { command.append(name) result = subprocess.run(command, capture_output=True) - response = result.stdout.decode('ascii') if result.returncode != 0: - print(response) + print( + f"Command: {command}\nOutput:\n" + + result.stdout.decode('utf-8') + + result.stderr.decode('utf-8'), + file=sys.stderr + ) raise RuntimeError(f'API request {name} failed') - return json.loads(response) + return result.stdout From db3beac9f27e5eb03665393008c76273188defaf Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Mon, 7 Jul 2025 23:48:41 +0200 Subject: [PATCH 03/12] Move predefined webapp dir into dj_utils.py Also initialize `domjudge_api_url = None` and simply call the API via CLI if `domjudge_api_url` is unset. --- misc-tools/.gitignore | 1 + misc-tools/Makefile | 3 ++- misc-tools/configure-domjudge.in | 10 +++------- misc-tools/{dj_utils.py => dj_utils.py.in} | 23 +++++++++++----------- misc-tools/import-contest.in | 9 +++------ 5 files changed, 21 insertions(+), 25 deletions(-) rename misc-tools/{dj_utils.py => dj_utils.py.in} (90%) diff --git a/misc-tools/.gitignore b/misc-tools/.gitignore index bb48b82358..da724f0b2e 100644 --- a/misc-tools/.gitignore +++ b/misc-tools/.gitignore @@ -5,4 +5,5 @@ /dj_make_chroot_docker /dj_run_chroot /dj_judgehost_cleanup +/dj_utils.py /force-passwords diff --git a/misc-tools/Makefile b/misc-tools/Makefile index eaa0d6345a..2a62c831e8 100644 --- a/misc-tools/Makefile +++ b/misc-tools/Makefile @@ -9,7 +9,8 @@ include $(TOPDIR)/Makefile.global TARGETS = OBJECTS = -SUBST_DOMSERVER = fix_permissions configure-domjudge import-contest force-passwords +SUBST_DOMSERVER = fix_permissions configure-domjudge dj_utils.py \ + import-contest force-passwords SUBST_JUDGEHOST = dj_make_chroot dj_run_chroot dj_make_chroot_docker \ dj_judgehost_cleanup diff --git a/misc-tools/configure-domjudge.in b/misc-tools/configure-domjudge.in index 360bb8e163..030c0862b0 100755 --- a/misc-tools/configure-domjudge.in +++ b/misc-tools/configure-domjudge.in @@ -24,8 +24,6 @@ from typing import List, Set sys.path.append('@domserver_libdir@') import dj_utils -webappdir = '@domserver_webappdir@' - def usage(): print(f'Usage: {sys.argv[0]} []') exit(1) @@ -57,11 +55,9 @@ def _keyify_list(l: List) -> Set: return { elem['id']: elem for elem in l } -if len(sys.argv) == 1: - dj_utils.domjudge_webapp_folder_or_api_url = webappdir -elif len(sys.argv) == 2: - dj_utils.domjudge_webapp_folder_or_api_url = sys.argv[1] -else: +if len(sys.argv) == 2: + dj_utils.domjudge_api_url = sys.argv[1] +elif len(sys.argv) != 1: usage() user_data = dj_utils.do_api_request('user') diff --git a/misc-tools/dj_utils.py b/misc-tools/dj_utils.py.in similarity index 90% rename from misc-tools/dj_utils.py rename to misc-tools/dj_utils.py.in index ab44b923de..bd1508898e 100644 --- a/misc-tools/dj_utils.py +++ b/misc-tools/dj_utils.py.in @@ -16,7 +16,8 @@ _myself = os.path.basename(sys.argv[0]) _default_user_agent = requests.utils.default_user_agent() headers = {'user-agent': f'dj_utils/{_myself} ({_default_user_agent})'} -domjudge_webapp_folder_or_api_url = 'unset' +domjudge_webapp_dir = '@domserver_webappdir@' +domjudge_api_url = None ca_check = True @@ -50,8 +51,8 @@ def parse_api_response(name: str, response: requests.Response) -> bytes: def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. - Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this - will use the DOMjudge CLI or HTTP API. + Based on whether `domjudge_api_url` is set, this will call the DOMjudge + API via HTTP or CLI. Parameters: name (str): the endpoint to call @@ -67,12 +68,12 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: cannot be JSON decoded. ''' - if os.path.isdir(domjudge_webapp_folder_or_api_url): + if domjudge_api_url is None: result = api_via_cli(name, method, {}, {}, jsonData) else: global ca_check - url = f'{domjudge_webapp_folder_or_api_url}/{name}' - parsed = urlparse(domjudge_webapp_folder_or_api_url) + url = f'{domjudge_api_url}/{name}' + parsed = urlparse(domjudge_api_url) auth = None if parsed.username and parsed.password: auth = (parsed.username, parsed.password) @@ -109,8 +110,8 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: def upload_file(name: str, apifilename: str, file: str, data: dict = {}): '''Upload the given file to the API at the given path with the given name. - Based on whether `domjudge_webapp_folder_or_api_url` is a folder or URL this - will use the DOMjudge CLI or HTTP API. + Based on whether `domjudge_api_url` is set, this will call the DOMjudge + API via HTTP or CLI. Parameters: name (str): the endpoint to call @@ -124,13 +125,13 @@ def upload_file(name: str, apifilename: str, file: str, data: dict = {}): RuntimeError when the HTTP status code is non 2xx. ''' - if os.path.isdir(domjudge_webapp_folder_or_api_url): + if domjudge_api_url is None: response = api_via_cli(name, 'POST', data, {apifilename: file}) else: global ca_check files = [(apifilename, open(file, 'rb'))] - url = f'{domjudge_webapp_folder_or_api_url}/{name}' + url = f'{domjudge_api_url}/{name}' try: response = requests.post( @@ -166,7 +167,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { ''' command = [ - f'{domjudge_webapp_folder_or_api_url}/bin/console', + f'{domjudge_webapp_dir}/bin/console', 'api:call', '-m', method diff --git a/misc-tools/import-contest.in b/misc-tools/import-contest.in index cb940e2334..bca6a5490a 100755 --- a/misc-tools/import-contest.in +++ b/misc-tools/import-contest.in @@ -29,7 +29,6 @@ sys.path.append('@domserver_libdir@') import dj_utils cid = None -webappdir = '@domserver_webappdir@' def usage(): @@ -134,11 +133,9 @@ def import_contest_problemset_document(cid: str): else: print('Skipping contest problemset import.') -if len(sys.argv) == 1: - dj_utils.domjudge_webapp_folder_or_api_url = webappdir -elif len(sys.argv) == 2: - dj_utils.domjudge_webapp_folder_or_api_url = sys.argv[1] -else: +if len(sys.argv) == 2: + dj_utils.domjudge_api_url = sys.argv[1] +elif len(sys.argv) != 1: usage() user_data = dj_utils.do_api_request('user') From c9bb97b861f3138cd8d6398d8c65631903dbb0db Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 00:50:52 +0200 Subject: [PATCH 04/12] Deduplicate code in upload_file and do_api_request --- misc-tools/dj_utils.py.in | 67 ++++++++++++++------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/misc-tools/dj_utils.py.in b/misc-tools/dj_utils.py.in index bd1508898e..777adae6ac 100644 --- a/misc-tools/dj_utils.py.in +++ b/misc-tools/dj_utils.py.in @@ -48,7 +48,8 @@ def parse_api_response(name: str, response: requests.Response) -> bytes: return response.content -def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: bool = True): +def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, + data: dict = {}, files: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. Based on whether `domjudge_api_url` is set, this will call the DOMjudge @@ -58,6 +59,7 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: name (str): the endpoint to call method (str): the method to use, GET or PUT are supported jsonData (dict): the JSON data to PUT. Only used when method is PUT + data (dict): data to pass in a POST request decode (bool): whether to decode the returned JSON data, default true Returns: @@ -78,13 +80,23 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: if parsed.username and parsed.password: auth = (parsed.username, parsed.password) + kwargs = { + 'headers': headers, + 'verify': ca_check, + 'auth': auth, + } + if method == 'GET': + pass + elif method == 'PUT': + kwargs['json'] = jsonData + elif method == 'POST': + kwargs['data'] = data + kwargs['files'] = [(name, open(file, 'rb')) for (name, file) in files] + else: + raise RuntimeError(f"Method {method} not supported") + try: - if method == 'GET': - response = requests.get(url, headers=headers, verify=ca_check, auth=auth) - elif method == 'PUT': - response = requests.put(url, headers=headers, verify=ca_check, auth=auth, json=jsonData) - else: - raise RuntimeError("Method not supported") + response = requests.request(method, url, **kwargs) except requests.exceptions.SSLError: ca_check = not confirm( "Can not verify certificate, ignore certificate check?", False) @@ -107,46 +119,13 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, decode: return result -def upload_file(name: str, apifilename: str, file: str, data: dict = {}): - '''Upload the given file to the API at the given path with the given name. - - Based on whether `domjudge_api_url` is set, this will call the DOMjudge - API via HTTP or CLI. - - Parameters: - name (str): the endpoint to call - apifilename (str): the argument name for the file to upload - file (str): the file to upload - - Returns: - The parsed endpoint contents. +def upload_file(endpoint: str, apifilename: str, file: str, data: dict = {}): + '''Upload the given file to the API endpoint as apifilename. - Raises: - RuntimeError when the HTTP status code is non 2xx. + Thin wrapper around do_api_request, see that function for more details. ''' - if domjudge_api_url is None: - response = api_via_cli(name, 'POST', data, {apifilename: file}) - else: - global ca_check - files = [(apifilename, open(file, 'rb'))] - - url = f'{domjudge_api_url}/{name}' - - try: - response = requests.post( - url, files=files, headers=headers, data=data, verify=ca_check) - except requests.exceptions.SSLError: - ca_check = not confirm( - "Can not verify certificate, ignore certificate check?", False) - if ca_check: - print('Can not verify certificate chain for DOMserver.') - exit(1) - else: - response = requests.post( - url, files=files, headers=headers, data=data, verify=ca_check) - - return parse_api_response(name, response) + do_api_request(endpoint, 'POST', data=data, files={apifilename: file}) def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {}, jsonData: dict = {}): From 56b916934e6859ed92cd9ff51c279f8550dc7b34 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 00:54:28 +0200 Subject: [PATCH 05/12] Rename parameter 'name' to 'endpoint' for clarity --- misc-tools/dj_utils.py.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/misc-tools/dj_utils.py.in b/misc-tools/dj_utils.py.in index 777adae6ac..a843700d2f 100644 --- a/misc-tools/dj_utils.py.in +++ b/misc-tools/dj_utils.py.in @@ -31,7 +31,7 @@ def confirm(message: str, default: bool) -> bool: return answer == 'y' -def parse_api_response(name: str, response: requests.Response) -> bytes: +def parse_api_response(endpoint: str, response: requests.Response) -> bytes: # The connection worked, but we may have received an HTTP error if response.status_code >= 300: print(response.text) @@ -40,7 +40,7 @@ def parse_api_response(name: str, response: requests.Response) -> bytes: 'Authentication failed, please check your DOMjudge credentials in ~/.netrc.') else: raise RuntimeError( - f'API request {name} failed (code {response.status_code}).') + f'API request {endpoint} failed (code {response.status_code}).') if response.status_code == 204: return None @@ -48,7 +48,7 @@ def parse_api_response(name: str, response: requests.Response) -> bytes: return response.content -def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, +def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, data: dict = {}, files: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. @@ -56,7 +56,7 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, API via HTTP or CLI. Parameters: - name (str): the endpoint to call + endpoint (str): the endpoint to call method (str): the method to use, GET or PUT are supported jsonData (dict): the JSON data to PUT. Only used when method is PUT data (dict): data to pass in a POST request @@ -71,10 +71,10 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, ''' if domjudge_api_url is None: - result = api_via_cli(name, method, {}, {}, jsonData) + result = api_via_cli(endpoint, method, {}, {}, jsonData) else: global ca_check - url = f'{domjudge_api_url}/{name}' + url = f'{domjudge_api_url}/{endpoint}' parsed = urlparse(domjudge_api_url) auth = None if parsed.username and parsed.password: @@ -107,14 +107,14 @@ def do_api_request(name: str, method: str = 'GET', jsonData: dict = {}, return do_api_request(name) except requests.exceptions.RequestException as e: raise RuntimeError(e) - result = parse_api_response(name, response) + result = parse_api_response(endpoint, response) if decode: try: result = json.loads(result) except json.decoder.JSONDecodeError as e: print(result) - raise RuntimeError(f'Failed to JSON decode the response for API request {name}') + raise RuntimeError(f'Failed to JSON decode the response for API request {endpoint}') return result @@ -128,11 +128,11 @@ def upload_file(endpoint: str, apifilename: str, file: str, data: dict = {}): do_api_request(endpoint, 'POST', data=data, files={apifilename: file}) -def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = {}, jsonData: dict = {}): +def api_via_cli(endpoint: str, method: str = 'GET', data: dict = {}, files: dict = {}, jsonData: dict = {}): '''Perform the given API request using the CLI Parameters: - name (str): the endpoint to call + endpoint (str): the endpoint to call, relative to API base url method (str): the method to use. Either GET, POST or PUT data (dict): the POST data to use. Only used when method is POST or PUT files (dict): the files to use. Only used when method is POST or PUT @@ -161,7 +161,7 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { if jsonData: command.extend(['-j', json.dumps(jsonData)]) - command.append(name) + command.append(endpoint) result = subprocess.run(command, capture_output=True) @@ -172,6 +172,6 @@ def api_via_cli(name: str, method: str = 'GET', data: dict = {}, files: dict = { result.stderr.decode('utf-8'), file=sys.stderr ) - raise RuntimeError(f'API request {name} failed') + raise RuntimeError(f'API request {endpoint} failed') return result.stdout From 16e9b1f54d77d4e8e53d8164c1429d4f6186e576 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 01:01:02 +0200 Subject: [PATCH 06/12] More cleanly retry without SSL validation Capture all function parameters and pass these as is to the function itself again, just now with the global `ca_check = False`. --- misc-tools/dj_utils.py.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misc-tools/dj_utils.py.in b/misc-tools/dj_utils.py.in index a843700d2f..ed385ed36a 100644 --- a/misc-tools/dj_utils.py.in +++ b/misc-tools/dj_utils.py.in @@ -70,6 +70,9 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, cannot be JSON decoded. ''' + # For the recursive call below to ignore SSL validation: + orig_kwargs = locals() + if domjudge_api_url is None: result = api_via_cli(endpoint, method, {}, {}, jsonData) else: @@ -104,7 +107,8 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, print('Can not verify certificate chain for DOMserver.') exit(1) else: - return do_api_request(name) + # Retry with SSL verification disabled + return do_api_request(**orig_kwargs) except requests.exceptions.RequestException as e: raise RuntimeError(e) result = parse_api_response(endpoint, response) From 4e3b52aec0f9f0a603b0dcbe12184d99ec2daa0b Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 01:03:21 +0200 Subject: [PATCH 07/12] Use `urljoin` to join base and relative path inside API This follows the specifications correctly, but beware that specifying a base URL that doesn't end with a `/` leads to strange results, see https://github.com/DOMjudge/domjudge/issues/2378 Replace `exit(1)` by an exception so this can be caught also if this code runs asynchronously in a thread or separate process. --- misc-tools/dj_utils.py.in | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/misc-tools/dj_utils.py.in b/misc-tools/dj_utils.py.in index ed385ed36a..aa55ba9838 100644 --- a/misc-tools/dj_utils.py.in +++ b/misc-tools/dj_utils.py.in @@ -11,7 +11,7 @@ import requests import requests.utils import subprocess import sys -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin _myself = os.path.basename(sys.argv[0]) _default_user_agent = requests.utils.default_user_agent() @@ -48,6 +48,13 @@ def parse_api_response(endpoint: str, response: requests.Response) -> bytes: return response.content +def is_relative(url: str) -> bool: + parsed = urlparse(url) + + return (parsed.scheme=='' and parsed.netloc=='' and + (len(parsed.path)==0 or parsed.path[0]!='/')) + + def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, data: dict = {}, files: dict = {}, decode: bool = True): '''Perform an API call to the given endpoint and return its data. @@ -56,7 +63,8 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, API via HTTP or CLI. Parameters: - endpoint (str): the endpoint to call + endpoint (str): the endpoint to call, relative to `domjudge_api_url`; + it may also contain a complete URL, which will then be used as-is method (str): the method to use, GET or PUT are supported jsonData (dict): the JSON data to PUT. Only used when method is PUT data (dict): data to pass in a POST request @@ -73,12 +81,14 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, # For the recursive call below to ignore SSL validation: orig_kwargs = locals() - if domjudge_api_url is None: + if domjudge_api_url is None and is_relative(endpoint): result = api_via_cli(endpoint, method, {}, {}, jsonData) else: + if not is_relative(endpoint): + raise RuntimeError(f'Cannot access non-relative URL {endpoint} without API base URL') global ca_check - url = f'{domjudge_api_url}/{endpoint}' parsed = urlparse(domjudge_api_url) + url = urljoin(domjudge_api_url, endpoint) auth = None if parsed.username and parsed.password: auth = (parsed.username, parsed.password) @@ -104,8 +114,7 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, ca_check = not confirm( "Can not verify certificate, ignore certificate check?", False) if ca_check: - print('Can not verify certificate chain for DOMserver.') - exit(1) + raise RuntimeError(f'Cannot verify certificate chain for {url}') else: # Retry with SSL verification disabled return do_api_request(**orig_kwargs) From d8db6cd5cbc122cb0c2643a5a7630fdb778998c4 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 23:35:59 +0200 Subject: [PATCH 08/12] Add option to save API output to file This is needed for binary content that otherwise gets mangled when fetched through the CLI. Also force all but the `endpoint` and `method` parameters to `do_api_request` to be named parameters to prevent mistakes. --- misc-tools/configure-domjudge.in | 2 +- misc-tools/dj_utils.py.in | 24 ++++++++++++++++----- webapp/src/Command/CallApiActionCommand.php | 14 +++++++++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/misc-tools/configure-domjudge.in b/misc-tools/configure-domjudge.in index 030c0862b0..b58660b5ac 100755 --- a/misc-tools/configure-domjudge.in +++ b/misc-tools/configure-domjudge.in @@ -84,7 +84,7 @@ if os.path.exists('config.json'): print(f' - missing keys from new config = {missing_keys}') if diffs or new_keys or missing_keys: if dj_utils.confirm(' - Upload these configuration changes?', True): - actual_config = dj_utils.do_api_request('config', 'PUT', expected_config) + actual_config = dj_utils.do_api_request('config', 'PUT', jsonData=expected_config) diffs, new_keys, missing_keys = compare_configs( actual_config=actual_config, expected_config=expected_config diff --git a/misc-tools/dj_utils.py.in b/misc-tools/dj_utils.py.in index aa55ba9838..5a93bb13de 100644 --- a/misc-tools/dj_utils.py.in +++ b/misc-tools/dj_utils.py.in @@ -55,8 +55,9 @@ def is_relative(url: str) -> bool: (len(parsed.path)==0 or parsed.path[0]!='/')) -def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, - data: dict = {}, files: dict = {}, decode: bool = True): +def do_api_request(endpoint: str, method: str = 'GET', *, + data: dict = {}, files: dict = {}, jsonData: dict = {}, + decode: bool = True, output_file: str = None): '''Perform an API call to the given endpoint and return its data. Based on whether `domjudge_api_url` is set, this will call the DOMjudge @@ -69,9 +70,12 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, jsonData (dict): the JSON data to PUT. Only used when method is PUT data (dict): data to pass in a POST request decode (bool): whether to decode the returned JSON data, default true + output_file (str): write API response to file, e.g. for binary content; + incompatible with decode=True. Returns: - The endpoint contents, either as raw bytes or JSON decoded. + The endpoint contents, either as raw bytes or JSON decoded, unless + output_file is specified. Raises: RuntimeError when the HTTP status code is non-2xx or the response @@ -82,7 +86,7 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, orig_kwargs = locals() if domjudge_api_url is None and is_relative(endpoint): - result = api_via_cli(endpoint, method, {}, {}, jsonData) + result = api_via_cli(endpoint, method, jsonData=jsonData, output_file=output_file) else: if not is_relative(endpoint): raise RuntimeError(f'Cannot access non-relative URL {endpoint} without API base URL') @@ -122,6 +126,10 @@ def do_api_request(endpoint: str, method: str = 'GET', jsonData: dict = {}, raise RuntimeError(e) result = parse_api_response(endpoint, response) + if output_file is not None: + with open(output_file, 'wb') as f: + f.write(result) + if decode: try: result = json.loads(result) @@ -141,7 +149,9 @@ def upload_file(endpoint: str, apifilename: str, file: str, data: dict = {}): do_api_request(endpoint, 'POST', data=data, files={apifilename: file}) -def api_via_cli(endpoint: str, method: str = 'GET', data: dict = {}, files: dict = {}, jsonData: dict = {}): +def api_via_cli(endpoint: str, method: str = 'GET', *, + data: dict = {}, files: dict = {}, jsonData: dict = {}, + output_file: str = None): '''Perform the given API request using the CLI Parameters: @@ -150,6 +160,7 @@ def api_via_cli(endpoint: str, method: str = 'GET', data: dict = {}, files: dict data (dict): the POST data to use. Only used when method is POST or PUT files (dict): the files to use. Only used when method is POST or PUT jsonData (dict): the JSON data to use. Only used when method is POST or PUT + output_file (str): write API response to file, e.g. for binary content Returns: The raw endpoint contents. @@ -174,6 +185,9 @@ def api_via_cli(endpoint: str, method: str = 'GET', data: dict = {}, files: dict if jsonData: command.extend(['-j', json.dumps(jsonData)]) + if output_file: + command.extend(['-o', output_file]) + command.append(endpoint) result = subprocess.run(command, capture_output=True) diff --git a/webapp/src/Command/CallApiActionCommand.php b/webapp/src/Command/CallApiActionCommand.php index 64a0fda581..a2a511dbb5 100644 --- a/webapp/src/Command/CallApiActionCommand.php +++ b/webapp/src/Command/CallApiActionCommand.php @@ -65,6 +65,12 @@ protected function configure(): void 'u', InputOption::VALUE_REQUIRED, 'User to use for API requests. If not given, the first admin user will be used' + ) + ->addOption( + 'output', + 'o', + InputOption::VALUE_REQUIRED, + 'Filename to write output to. Useful for binary content' ); } @@ -154,7 +160,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $output->writeln($response); + if ($filename = $input->getOption('output')) { + $fd = fopen($filename, 'w'); + fwrite($fd, $response); + fclose($fd); + } else { + $output->write($response); + } return Command::SUCCESS; } } From 7ffdd9bee90533f3f00c4fc587d1c70b2431f8c2 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 8 Jul 2025 23:54:43 +0200 Subject: [PATCH 09/12] Handle BinaryFileResponse in API CLI correctly --- webapp/src/Service/DOMJudgeService.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp/src/Service/DOMJudgeService.php b/webapp/src/Service/DOMJudgeService.php index 5e8a92ca40..eac465d07c 100644 --- a/webapp/src/Service/DOMJudgeService.php +++ b/webapp/src/Service/DOMJudgeService.php @@ -43,6 +43,7 @@ use ReflectionClass; use Symfony\Component\DependencyInjection\Attribute\Autowire; use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; +use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\Cookie; use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\HttpFoundation\InputBag; @@ -652,7 +653,8 @@ public function internalApiRequest(string $url, string $method = Request::METHOD return null; } - if ($response instanceof StreamedResponse) { + if ($response instanceof StreamedResponse || + $response instanceof BinaryFileResponse) { ob_start(flags: PHP_OUTPUT_HANDLER_REMOVABLE); $response->sendContent(); $content = ob_get_clean(); From 2c3d5042fdb8bbea06f188c77c99566b0687a9b0 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Mon, 14 Jul 2025 23:38:54 +0200 Subject: [PATCH 10/12] Use ArgumentParser and make arguments optional This gives more flexibility for extension in the future and leaves the default usage without arguments unchanged. --- misc-tools/import-contest.in | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/misc-tools/import-contest.in b/misc-tools/import-contest.in index bca6a5490a..53201f6a80 100755 --- a/misc-tools/import-contest.in +++ b/misc-tools/import-contest.in @@ -15,7 +15,8 @@ Part of the DOMjudge Programming Contest Jury System and licensed under the GNU GPL. See README and COPYING for details. ''' -from os import listdir +from argparse import ArgumentParser +from os import chdir, listdir from typing import List import json import os.path @@ -30,11 +31,15 @@ import dj_utils cid = None +parser = ArgumentParser(description='Import a contest archive to DOMjudge via the API.') +parser.add_argument('-d', '--dir', help="directory containing the contest archive, defaults to current directory") +parser.add_argument('-u', '--url', help="DOMjudge API URL to use, if not specified use the CLI interface") +args = parser.parse_args() -def usage(): - print(f'Usage: {sys.argv[0]} []') - exit(1) - +if args.dir: + chdir(args.dir) +if args.url: + dj_utils.domjudge_api_url = args.url def import_file(entity: str, files: List[str]) -> bool: any_matched = False @@ -133,11 +138,6 @@ def import_contest_problemset_document(cid: str): else: print('Skipping contest problemset import.') -if len(sys.argv) == 2: - dj_utils.domjudge_api_url = sys.argv[1] -elif len(sys.argv) != 1: - usage() - user_data = dj_utils.do_api_request('user') if 'admin' not in user_data['roles']: print('Your user does not have the \'admin\' role, can not import.') From d13e64399ac47cbf9da2cb22a3e1d9a2f2e38339 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 6 Jul 2025 16:00:25 +0200 Subject: [PATCH 11/12] Add contest-export script --- misc-tools/Makefile | 2 +- misc-tools/export-contest.in | 190 +++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 1 deletion(-) create mode 100755 misc-tools/export-contest.in diff --git a/misc-tools/Makefile b/misc-tools/Makefile index 2a62c831e8..d216c138cc 100644 --- a/misc-tools/Makefile +++ b/misc-tools/Makefile @@ -10,7 +10,7 @@ TARGETS = OBJECTS = SUBST_DOMSERVER = fix_permissions configure-domjudge dj_utils.py \ - import-contest force-passwords + import-contest export-contest force-passwords SUBST_JUDGEHOST = dj_make_chroot dj_run_chroot dj_make_chroot_docker \ dj_judgehost_cleanup diff --git a/misc-tools/export-contest.in b/misc-tools/export-contest.in new file mode 100755 index 0000000000..39957b93c5 --- /dev/null +++ b/misc-tools/export-contest.in @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +''' +export-contest -- Convenience script to export a contest (including metadata, +teams and problems) from the command line. Defaults to using the CLI interface; +Specify a DOMjudge API URL as to use that. + +Reads credentials from ~/.netrc when using the API. + +Part of the DOMjudge Programming Contest Jury System and licensed +under the GNU GPL. See README and COPYING for details. +''' + +import datetime +import json +import os +import sys +import time +from argparse import ArgumentParser +from concurrent.futures import ThreadPoolExecutor, as_completed + +sys.path.append('@domserver_libdir@') +import dj_utils + +mime_to_extension = { + 'application/pdf': 'pdf', + 'application/zip': 'zip', + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/svg+xml': 'svg', + 'text/plain': 'txt', + 'video/mp4': 'mp4', + 'video/mpeg': 'mpg', + 'video/webm': 'webm', +} + +def get_default_contest(): + c_default = None + + contests = dj_utils.do_api_request('contests') + if len(contests)>0: + now = int(time.time()) + for c in contests: + if 'start_time' not in c or c['start_time'] is None: + # Assume that a contest with start time unset will start soon. + c['start_epoch'] = now + 1 + else: + c['start_epoch'] = datetime.datetime.fromisoformat(c['start_time']).timestamp() + + c_default = contests[0] + for c in contests: + if c_default['start_epoch']<=now: + if c['start_epoch']<=now and c['start_epoch']>c_default['start_epoch']: + c_default = c + else: + if c['start_epoch'] bool: + ''' + Check whether API data represents a FILE object. This is heuristic because + no property is strictly required, but we need at least `href` to download + the file, so if also we find one other property, we announce a winner. + ''' + if not isinstance(data, dict): + return false + return 'href' in data and ('mime' in data or 'filename' in data or 'hash' in data) + + +files_to_download = [] + +def recurse_find_files(data, store_path: str, default_name: str): + if isinstance(data, list): + # Special case single element list for simpler default_name + if len(data) == 1: + recurse_find_files(data[0], store_path, default_name) + else: + for i, item in enumerate(data): + recurse_find_files(item, store_path, f"{default_name}.{i}") + elif isinstance(data, dict): + if is_file(data): + if 'mime' in data and data['mime'] in mime_to_extension: + default_name += '.' + mime_to_extension[data['mime']] + files_to_download.append((data, store_path, default_name)) + else: + for key, item in data.items(): + recurse_find_files(item, store_path, f"{default_name}.{key}") + + +def download_endpoint(name: str, path: str): + ext = '.ndjson' if name == 'event-feed' else '.json' + filename = name + ext + + print(f"Fetching '{path}' to '{filename}'") + data = dj_utils.do_api_request(path, decode=False) + with open(filename, 'wb') as f: + f.write(data) + + if ext == '.json': + data = json.loads(data) + store_path = name + if isinstance(data, list): + for elem in data: + recurse_find_files(elem, f"{store_path}/{elem['id']}", '') + else: + recurse_find_files(data, store_path, '') + + +cid = None +dir = None + +parser = ArgumentParser(description='Export a contest archive from DOMjudge via the API.') +parser.add_argument('-c', '--cid', help="contest ID to export, defaults to last started, or else first non-started active contest") +parser.add_argument('-d', '--dir', help="directory to write the contest archive to, defaults to contest ID in current directory") +parser.add_argument('-u', '--url', help="DOMjudge API URL to use, if not specified use the CLI interface") +args = parser.parse_args() + +if args.cid: + cid = args.cid +else: + c = get_default_contest() + if c is None: + print("No contest specified nor an active contest found.") + exit(1) + else: + cid = c['id'] + +if args.dir: + dir = args.dir +else: + dir = cid + +if args.url: + dj_utils.domjudge_api_url = args.url + +user_data = dj_utils.do_api_request('user') +if 'admin' not in user_data['roles']: + print('Your user does not have the \'admin\' role, can not export.') + exit(1) + +if os.path.exists(dir): + print(f'Export directory \'{dir}\' already exists, will not overwrite.') + exit(1) + +os.makedirs(dir) +os.chdir(dir) + +contest_path = f'contests/{cid}' + +# Custom endpoints: +download_endpoint('api', '') +download_endpoint('contest', contest_path) +download_endpoint('event-feed', f'{contest_path}/event-feed?stream=false') + +for endpoint in [ + 'access', + 'accounts', + 'awards', +# 'balloons', This is a DOMjudge specific endpoint + 'clarifications', +# 'commentary', Not implemented in DOMjudge + 'groups', + 'judgement-types', + 'judgements', + 'languages', + 'organizations', +# 'persons', Not implemented in DOMjudge + 'problems', + 'runs', + 'scoreboard', + 'state', + 'submissions', + 'teams', + ]: + download_endpoint(endpoint, f"{contest_path}/{endpoint}") + +with ThreadPoolExecutor(20) as executor: + futures = [executor.submit(download_file, *item) for item in files_to_download] + for future in as_completed(futures): + future.result() # So it can throw any exception From adb08ee50e50f95e762d6873f477a906d12b477b Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Tue, 15 Jul 2025 23:01:33 +0200 Subject: [PATCH 12/12] Fix variable rename Co-authored-by: Nicky Gerritsen --- webapp/src/Service/EventLogService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/Service/EventLogService.php b/webapp/src/Service/EventLogService.php index c31ad90a10..257afc358f 100644 --- a/webapp/src/Service/EventLogService.php +++ b/webapp/src/Service/EventLogService.php @@ -315,7 +315,7 @@ public function log( $query = ['ids' => $ids]; } - $this->dj->withAllRoles(function () use ($query, $url, &$json) { + $this->dj->withAllRoles(function () use ($query, $url, &$response) { $response = $this->dj->internalApiRequest($url, Request::METHOD_GET, $query); });