From 6c019510831f4f502f423b2c57a44564176eb027 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Thu, 29 Feb 2024 03:11:51 +0530 Subject: [PATCH 1/8] feat: Add script to check updates for dependency daily. --- scripts/check_for_update_dependency.py | 74 ++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 scripts/check_for_update_dependency.py diff --git a/scripts/check_for_update_dependency.py b/scripts/check_for_update_dependency.py new file mode 100644 index 000000000000..cc537c894566 --- /dev/null +++ b/scripts/check_for_update_dependency.py @@ -0,0 +1,74 @@ +import re +import requests +import toml +from typing import Optional + + +def get_latest_version(package_name: str) -> str: + """Fetch the latest version of a package from PyPI.""" + response = requests.get(f"https://pypi.org/pypi/{package_name}/json") + if response.status_code == 200: + data = response.json() + return data['info']['version'] + else: + raise Exception( + f"Failed to fetch the latest version for {package_name}") + + +def update_versions(file_path: str) -> None: + # Read the current content of the file + with open(file_path, "r") as file: + content = file.readlines() + + # Pattern to match the packages of interest and capture their current versions + pattern = re.compile(r"^(pytype|mypy)==([\d.]+)") + + # Updated content + updated_content = [] + + for line in content: + match = pattern.match(line) + if match: + package_name = match.group(1) + current_version = match.group(2) + try: + latest_version = get_latest_version(package_name) + if current_version != latest_version: + print( + f"Updating {package_name} from {current_version} to {latest_version}") + line = line.replace(current_version, latest_version) + except Exception as e: + print(f"Error updating {package_name}: {e}") + updated_content.append(line) + + # Write the updated content back to the file + with open(file_path, "w") as file: + file.writelines(updated_content) + + +def update_pyright_version(file_path: str) -> None: + # Load the TOML file + with open(file_path, "r", encoding="utf-8") as file: + config = toml.load(file) + + # Get the latest version of pyright + latest_version = get_latest_version("pyright") + + if latest_version: + # Update the pyright version in the configuration + config["tool"]["typeshed"]["pyright_version"] = latest_version + + # Write the updated configuration back to the file + with open(file_path, "w", encoding="utf-8") as file: + toml.dump(config, file) + print(f"Updated pyright to version {latest_version} in {file_path}") + else: + print("No update performed.") + + +# Path to your file +file_path: list[str] = ["requirements-tests.txt", "pyproject.toml"] + +if __name__ == "__main__": + update_versions(file_path[0]) + update_pyright_version(file_path[1]) From a9d57967269f182a64066d57584423c1bfb48b7f Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Thu, 29 Feb 2024 03:12:29 +0530 Subject: [PATCH 2/8] Feat: Add a github action to check for dependency updates daily and create a PR with updated deps. --- .../workflows/update_daily_ dependency.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/update_daily_ dependency.yml diff --git a/.github/workflows/update_daily_ dependency.yml b/.github/workflows/update_daily_ dependency.yml new file mode 100644 index 000000000000..962b3d416b1a --- /dev/null +++ b/.github/workflows/update_daily_ dependency.yml @@ -0,0 +1,34 @@ +name: Check Dependencies + +on: + schedule: + # Runs every day at 00:00 UTC + - cron: '0 0 * * *' + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install requests + pip install toml + - name: Run script + run: python scripts/update_deps.py + - uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.PAT }} + title: "Update dpendency Daily" + commit-message: "Updated Depenceny after the daily check" + body: | + There appear to be some dependency updates in ${{ github.sha }}. This pull request + uses the script to find and update these dependency. + base: ${{ github.head_ref }} \ No newline at end of file From b90ac3dfb75552bf75cbaa153920ba69e20eb708 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Thu, 29 Feb 2024 03:14:29 +0530 Subject: [PATCH 3/8] fix: fixed the script name in actions. --- .github/workflows/update_daily_ dependency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_daily_ dependency.yml b/.github/workflows/update_daily_ dependency.yml index 962b3d416b1a..7d0b273ab35e 100644 --- a/.github/workflows/update_daily_ dependency.yml +++ b/.github/workflows/update_daily_ dependency.yml @@ -22,7 +22,7 @@ jobs: pip install requests pip install toml - name: Run script - run: python scripts/update_deps.py + run: python scripts/check_for_update_dependency.py - uses: peter-evans/create-pull-request@v3 with: token: ${{ secrets.PAT }} From 4370ef958207f074332957eb570dc60133a1a60e Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Thu, 29 Feb 2024 03:21:58 +0530 Subject: [PATCH 4/8] reformat: Use black to reformat. --- scripts/check_for_update_dependency.py | 43 ++++++++++---------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/scripts/check_for_update_dependency.py b/scripts/check_for_update_dependency.py index cc537c894566..20b2bd147439 100644 --- a/scripts/check_for_update_dependency.py +++ b/scripts/check_for_update_dependency.py @@ -1,74 +1,63 @@ import re import requests import toml +from typing import List + from typing import Optional -def get_latest_version(package_name: str) -> str: +def get_latest_version(package_name: str) -> Optional[str]: """Fetch the latest version of a package from PyPI.""" response = requests.get(f"https://pypi.org/pypi/{package_name}/json") if response.status_code == 200: data = response.json() - return data['info']['version'] + return str(data["info"]["version"]) else: - raise Exception( - f"Failed to fetch the latest version for {package_name}") + return None def update_versions(file_path: str) -> None: - # Read the current content of the file with open(file_path, "r") as file: content = file.readlines() - # Pattern to match the packages of interest and capture their current versions pattern = re.compile(r"^(pytype|mypy)==([\d.]+)") - # Updated content - updated_content = [] + updated_content: List[str] = [] for line in content: match = pattern.match(line) if match: package_name = match.group(1) current_version = match.group(2) - try: - latest_version = get_latest_version(package_name) - if current_version != latest_version: - print( - f"Updating {package_name} from {current_version} to {latest_version}") - line = line.replace(current_version, latest_version) - except Exception as e: - print(f"Error updating {package_name}: {e}") + latest_version = get_latest_version(package_name) + if latest_version and current_version != latest_version: + print(f"Updating {package_name} from {current_version} to {latest_version}") + line = line.replace(current_version, latest_version) updated_content.append(line) - # Write the updated content back to the file with open(file_path, "w") as file: file.writelines(updated_content) def update_pyright_version(file_path: str) -> None: - # Load the TOML file with open(file_path, "r", encoding="utf-8") as file: config = toml.load(file) - # Get the latest version of pyright latest_version = get_latest_version("pyright") if latest_version: - # Update the pyright version in the configuration config["tool"]["typeshed"]["pyright_version"] = latest_version - # Write the updated configuration back to the file with open(file_path, "w", encoding="utf-8") as file: toml.dump(config, file) print(f"Updated pyright to version {latest_version} in {file_path}") - else: - print("No update performed.") -# Path to your file -file_path: list[str] = ["requirements-tests.txt", "pyproject.toml"] +file_paths: List[str] = ["requirements-tests.txt", "pyproject.toml"] if __name__ == "__main__": - update_versions(file_path[0]) - update_pyright_version(file_path[1]) + for file_path in file_paths: + if file_path.endswith(".txt"): + update_versions(file_path) + elif file_path.endswith(".toml"): + update_pyright_version(file_path) From 26d9424a6956951661cf0ddec389f2fddb97e169 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Mon, 4 Mar 2024 13:44:17 +0530 Subject: [PATCH 5/8] feat: Improved Error Handling and Preserve File Integrity --- scripts/check_for_update_dependency.py | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/check_for_update_dependency.py b/scripts/check_for_update_dependency.py index 20b2bd147439..cd99efc0ad8c 100644 --- a/scripts/check_for_update_dependency.py +++ b/scripts/check_for_update_dependency.py @@ -1,6 +1,5 @@ import re import requests -import toml from typing import List from typing import Optional @@ -9,11 +8,9 @@ def get_latest_version(package_name: str) -> Optional[str]: """Fetch the latest version of a package from PyPI.""" response = requests.get(f"https://pypi.org/pypi/{package_name}/json") - if response.status_code == 200: - data = response.json() - return str(data["info"]["version"]) - else: - return None + response.raise_for_status() + data = response.json() + return str(data["info"]["version"]) def update_versions(file_path: str) -> None: @@ -31,7 +28,8 @@ def update_versions(file_path: str) -> None: current_version = match.group(2) latest_version = get_latest_version(package_name) if latest_version and current_version != latest_version: - print(f"Updating {package_name} from {current_version} to {latest_version}") + print( + f"Updating {package_name} from {current_version} to {latest_version}") line = line.replace(current_version, latest_version) updated_content.append(line) @@ -40,17 +38,23 @@ def update_versions(file_path: str) -> None: def update_pyright_version(file_path: str) -> None: + """Update pyright version in a TOML configuration file.""" with open(file_path, "r", encoding="utf-8") as file: - config = toml.load(file) + lines = file.readlines() latest_version = get_latest_version("pyright") - if latest_version: - config["tool"]["typeshed"]["pyright_version"] = latest_version + updated_lines = [] + for line in lines: + if line.startswith("pyright_version = "): + updated_lines.append(f'pyright_version = "{latest_version}"\n') + print( + f"Updated pyright to version {latest_version} in {file_path}") + else: + updated_lines.append(line) with open(file_path, "w", encoding="utf-8") as file: - toml.dump(config, file) - print(f"Updated pyright to version {latest_version} in {file_path}") + file.writelines(updated_lines) file_paths: List[str] = ["requirements-tests.txt", "pyproject.toml"] From c76fc4dd9679bb899252141ef0fcd4b01ecacbae Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Mon, 4 Mar 2024 15:07:13 +0530 Subject: [PATCH 6/8] fix: Typos and add create-issue-on-failure --- .../workflows/update_daily_ dependency.yml | 34 ------------- .github/workflows/update_mypy_pyright.yml | 51 +++++++++++++++++++ 2 files changed, 51 insertions(+), 34 deletions(-) delete mode 100644 .github/workflows/update_daily_ dependency.yml create mode 100644 .github/workflows/update_mypy_pyright.yml diff --git a/.github/workflows/update_daily_ dependency.yml b/.github/workflows/update_daily_ dependency.yml deleted file mode 100644 index 7d0b273ab35e..000000000000 --- a/.github/workflows/update_daily_ dependency.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Check Dependencies - -on: - schedule: - # Runs every day at 00:00 UTC - - cron: '0 0 * * *' - -jobs: - build: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install requests - pip install toml - - name: Run script - run: python scripts/check_for_update_dependency.py - - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.PAT }} - title: "Update dpendency Daily" - commit-message: "Updated Depenceny after the daily check" - body: | - There appear to be some dependency updates in ${{ github.sha }}. This pull request - uses the script to find and update these dependency. - base: ${{ github.head_ref }} \ No newline at end of file diff --git a/.github/workflows/update_mypy_pyright.yml b/.github/workflows/update_mypy_pyright.yml new file mode 100644 index 000000000000..72a2c94d7078 --- /dev/null +++ b/.github/workflows/update_mypy_pyright.yml @@ -0,0 +1,51 @@ +name: Update mypy, pytype and pyright + +on: + schedule: + # Runs every day at 00:00 UTC + - cron: '0 0 * * *' + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install requests + pip install toml + - name: Run script + run: python scripts/check_for_update_dependency.py + - uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Daily dependencies update" + commit-message: "Updated dependencies after the daily check" + body: | + There appear to be some dependency updates in ${{ github.sha }}. This pull request + uses the script to find and update these dependencies. + base: ${{ github.head_ref }} + # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2 + create-issue-on-failure: + name: Create an issue if Update mypy, pytype and pyright failed + runs-on: ubuntu-latest + needs: [build] + if: ${{ github.repository == 'python/typeshed' && always() && (needs.build.result == 'failure') }} + steps: + - uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.create({ + owner: "python", + repo: "typeshed", + title: `Update mypy, pytype and pyright failed on ${new Date().toDateString()}`, + body: "Update mypy, pytype and pyright runs are listed here: https://github.com/python/typeshed/actions/workflows/update_mypy_pyright.yml", + }) \ No newline at end of file From 42eb297e71ecd36e22d312b9fcfb49a5d8e2eaf7 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Mon, 4 Mar 2024 15:10:43 +0530 Subject: [PATCH 7/8] chore: remove unused dependencies --- .github/workflows/update_mypy_pyright.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update_mypy_pyright.yml b/.github/workflows/update_mypy_pyright.yml index 72a2c94d7078..e5c2c561ee0e 100644 --- a/.github/workflows/update_mypy_pyright.yml +++ b/.github/workflows/update_mypy_pyright.yml @@ -20,7 +20,6 @@ jobs: - name: Install dependencies run: | pip install requests - pip install toml - name: Run script run: python scripts/check_for_update_dependency.py - uses: peter-evans/create-pull-request@v3 From 3d4ff603b82b1b8e7515f9f0fb5f31b78c1c1f50 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Mon, 4 Mar 2024 15:11:11 +0530 Subject: [PATCH 8/8] refactor: run black formatter --- scripts/check_for_update_dependency.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/check_for_update_dependency.py b/scripts/check_for_update_dependency.py index cd99efc0ad8c..75d90aeb2428 100644 --- a/scripts/check_for_update_dependency.py +++ b/scripts/check_for_update_dependency.py @@ -28,8 +28,7 @@ def update_versions(file_path: str) -> None: current_version = match.group(2) latest_version = get_latest_version(package_name) if latest_version and current_version != latest_version: - print( - f"Updating {package_name} from {current_version} to {latest_version}") + print(f"Updating {package_name} from {current_version} to {latest_version}") line = line.replace(current_version, latest_version) updated_content.append(line) @@ -48,8 +47,7 @@ def update_pyright_version(file_path: str) -> None: for line in lines: if line.startswith("pyright_version = "): updated_lines.append(f'pyright_version = "{latest_version}"\n') - print( - f"Updated pyright to version {latest_version} in {file_path}") + print(f"Updated pyright to version {latest_version} in {file_path}") else: updated_lines.append(line)