From 9978f54afa224c529c0bc22208e961c5991d6853 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 11 Apr 2023 10:09:00 -0700 Subject: [PATCH 1/3] Update testing url to testing.psij.io --- testing.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing.conf b/testing.conf index 00657cc5..73821a6e 100644 --- a/testing.conf +++ b/testing.conf @@ -130,7 +130,7 @@ custom_attributes = # # server_url = -server_url = https://psij.testing.exaworks.org +server_url = https://testing.psij.io # From 7be22f7bf92d0fedc1be33fc22c5b3743192fdf7 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 11 Apr 2023 10:09:40 -0700 Subject: [PATCH 2/3] Updated CI runner to patch the testing URL. Also, added a more formalized way of patching things. --- tests/ci_runner.py | 111 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/tests/ci_runner.py b/tests/ci_runner.py index f6447c06..fcf129b6 100755 --- a/tests/ci_runner.py +++ b/tests/ci_runner.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, TextIO, List, Optional, Generator +from typing import Dict, TextIO, List, Optional, Generator, Callable import requests @@ -17,6 +17,7 @@ FAKE_BRANCHES = ['main', 'feature_1', 'feature_x'] GITHUB_API_ROOT = 'https://api.github.com' MODE = 'plain' +TARGET_PATCH_LEVEL = 2 def read_line(f: TextIO) -> Optional[str]: @@ -170,39 +171,96 @@ def run_tests(conf: Dict[str, str], site_ids: List[str], branches: List[str], cl install_deps(branch, tmpp) with info('Testing branch "%s"' % branch): run_branch_tests(conf, tmpp, run_id, clone, site_id, branch) +# Patching routines -OLD_REPO = 'ExaWorks/psi-j-python' -NEW_REPO = 'ExaWorks/psij-python' +def write_patch_level(level: int) -> None: + with open('.ci.patchlevel', 'w') as f: + f.write(str(level)) -def patch_file(file_name: str) -> None: - if os.path.exists(file_name + '.is_patched'): +def current_patch_level() -> int: + try: + with open('.ci.patchlevel') as f: + return int(f.read().strip()) + except OSError: + for fn in ['testing.conf', 'psij-ci-run']: + if not Path(fn + '.is_patched').exists(): + return 0 + write_patch_level(1) + return 1 + + +def deploy_patch(level: int) -> None: + if level == 1: + l1_patch_repo() + l1_update_origin() + elif level == 2: + l2_remove_patch_flag_files() + l2_update_upload_url() + else: + raise Exception('Nothing to do for patch level %s' % level) + write_patch_level(level) + + +def try_patch(level: int) -> None: + if level <= current_patch_level(): return + else: + deploy_patch(level) + + +def deploy_patches() -> None: + for level in range(1, TARGET_PATCH_LEVEL + 1): + try_patch(level) + +def line_patcher(file_name: str, matcher: Callable[[str], bool], + mutator: Callable[[str], str]) -> None: with info('Patching %s' % file_name): with open(file_name) as inf: with open(file_name + '._new_', 'w') as outf: for line in inf: # strip new line - if line.find(OLD_REPO) != -1: + if matcher(line): # we're adding one space so that the line has the same length; # when invoking a subprocess, bash stores the location where # it's supposed to continue parsing from, so it's a good idea # to to not move things around - line = line.rstrip('\n').replace(OLD_REPO, NEW_REPO) + ' \n' + line = mutator(line) outf.write(line) os.chmod(file_name + '._new_', os.stat(file_name).st_mode) os.rename(file_name + '._new_', file_name) - Path(file_name + '.is_patched').touch() -def patch_repo() -> None: - patch_file('testing.conf') - patch_file('psij-ci-run') +# Patch 1 +# Updates repositories in testing.conf and psij-ci-run. It also +# upates the git origin to point to the new repo. This is done to +# account for the fact that we renamed the repo from psi-j-python +# to psij-python. + +OLD_REPO = 'ExaWorks/psi-j-python' +NEW_REPO = 'ExaWorks/psij-python' + + +def l1_patch_file(file_name: str) -> None: + if os.path.exists(file_name + '.is_patched'): + return + line_patcher(file_name, + lambda line: line.find(OLD_REPO) != -1, + # The extra space before the newline is to not shift the content + # of psij-ci-run that follows this line. Bash continues reading + # the file after a command completes, but, if the content has + # shifted, it might end up reading a partial line. + lambda line: line.rstrip('\n').replace(OLD_REPO, NEW_REPO) + ' \n') -def update_origin() -> None: +def l1_patch_repo() -> None: + l1_patch_file('testing.conf') + l1_patch_file('psij-ci-run') + + +def l1_update_origin() -> None: old_url = run('git', 'config', '--get', 'remote.origin.url') new_url = old_url.strip().replace(OLD_REPO, NEW_REPO) if new_url != old_url: @@ -210,6 +268,32 @@ def update_origin() -> None: run('git', 'remote', 'set-url', 'origin', new_url) +# Patch 2 +# Updates the test upload url from either testing.exaworks.org or +# psij.testing.exaworks.org to testing.psij.io + +OLD_UPLOAD_URLS = ['https://psij.testing.exaworks.org', 'https://testing.exaworks.org'] +NEW_UPLOAD_URL = 'https://testing.psij.io' + + +def l2_remove_patch_flag_files() -> None: + # we're using a single patch level file now + for fn in ['testing.conf', 'psij-ci-run']: + f = Path(fn + '.is_patched') + if f.exists(): + f.unlink() + + +def l2_update_upload_url() -> None: + for old_url in OLD_UPLOAD_URLS: + line_patcher('testing.conf', + lambda line: line.find('server_url') != -1 + and line.find(old_url) != -1, # noqa: E127 + lambda line: line.rstrip('\n').replace(old_url, NEW_UPLOAD_URL) + '\n') + +# End of patches + + @contextmanager def info(msg: str) -> Generator[bool, None, None]: print(msg + '... ', end='') @@ -246,6 +330,5 @@ def info(msg: str) -> Generator[bool, None, None]: else: raise ValueError('Unrecognized value for scope: "%s"' % scope) - patch_repo() - update_origin() + deploy_patches() run_tests(conf, site_ids, branches, clone) From dab10ef6f322a65b89483758959446c2dfeeb5a3 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 11 Apr 2023 10:24:00 -0700 Subject: [PATCH 3/3] Spelling --- tests/ci_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_runner.py b/tests/ci_runner.py index fcf129b6..5dfeca1c 100755 --- a/tests/ci_runner.py +++ b/tests/ci_runner.py @@ -235,7 +235,7 @@ def line_patcher(file_name: str, matcher: Callable[[str], bool], # Patch 1 # Updates repositories in testing.conf and psij-ci-run. It also -# upates the git origin to point to the new repo. This is done to +# updates the git origin to point to the new repo. This is done to # account for the fact that we renamed the repo from psi-j-python # to psij-python.