Development longregression

.binder/apt.txt

@@ -0,0 +1,2 @@
+build-essential
+swig

.binder/postBuild

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+python -m pip install .[docs,examples]
+
+# Taken from https://github.com/scikit-learn/scikit-learn/blob/22cd233e1932457947e9994285dc7fd4e93881e4/.binder/postBuild
+# under BSD3 license, copyright the scikit-learn contributors
+
+# This script is called in a binder context. When this script is called, we are
+# inside a git checkout of the automl/Auto-PyTorch repo. This script
+# generates notebooks from the Auto-PyTorch python examples.
+
+if [[ ! -f /.dockerenv ]]; then
+    echo "This script was written for repo2docker and is supposed to run inside a docker container."
+    echo "Exiting because this script can delete data if run outside of a docker container."
+    exit 1
+fi
+
+# Copy content we need from the Auto-PyTorch repo
+TMP_CONTENT_DIR=/tmp/Auto-PyTorch
+mkdir -p $TMP_CONTENT_DIR
+cp -r examples .binder $TMP_CONTENT_DIR
+# delete everything in current directory including dot files and dot folders
+find . -delete
+
+# Generate notebooks and remove other files from examples folder
+GENERATED_NOTEBOOKS_DIR=examples
+cp -r $TMP_CONTENT_DIR/examples $GENERATED_NOTEBOOKS_DIR
+
+find $GENERATED_NOTEBOOKS_DIR -name 'example_*.py' -exec sphx_glr_python_to_jupyter.py '{}' +
+# Keep __init__.py and custom_metrics.py
+NON_NOTEBOOKS=$(find $GENERATED_NOTEBOOKS_DIR -type f | grep -v '\.ipynb' | grep -v 'init' | grep -v 'custom_metrics')
+rm -f $NON_NOTEBOOKS
+
+# Modify path to be consistent by the path given by sphinx-gallery
+mkdir notebooks
+mv $GENERATED_NOTEBOOKS_DIR notebooks/
+
+# Put the .binder folder back (may be useful for debugging purposes)
+mv $TMP_CONTENT_DIR/.binder .
+# Final clean up
+rm -rf $TMP_CONTENT_DIR

.binder/requirements.txt

@@ -0,0 +1 @@
+-r ../requirements.txt

.codecov.yml

@@ -0,0 +1,42 @@
+#see https://github.com/codecov/support/wiki/Codecov-Yaml
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  precision: 2  # 2 = xx.xx%, 0 = xx%
+  round: nearest # how coverage is rounded: down/up/nearest
+  range: 10...90 # custom range of coverage colors from red -> yellow -> green
+  status:
+    # https://codecov.readme.io/v1.0/docs/commit-status
+    project:
+      default:
+        against: auto
+        target: 70% # specify the target coverage for each commit status
+        threshold: 50% # allow this little decrease on project
+        # https://github.com/codecov/support/wiki/Filtering-Branches
+        # branches: master
+        if_ci_failed: error
+    # https://github.com/codecov/support/wiki/Patch-Status
+    patch:
+      default:
+        against: auto
+        target: 30% # specify the target "X%" coverage to hit
+        threshold: 50% # allow this much decrease on patch
+    changes: false
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: true
+      loop: true
+      macro: false
+      method: false
+  javascript:
+    enable_partials: false
+
+comment:
+  layout: header, diff
+  require_changes: false
+  behavior: default  # update if exists else create new
+  branches: *

.coveragerc

@@ -0,0 +1,27 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+include = "autoPyTorch/*"
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ max-line-length = 120
+ show-source = True
+ application-import-names = autoPyTorch
+ exclude =
+     venv
+     build

.github/workflows/dist.yml

@@ -0,0 +1,31 @@
+name: dist-check
+
+on: [push, pull_request]
+
+jobs:
+  dist:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Build dist
+      run: |
+        python setup.py sdist
+    - name: Twine check
+      run: |
+        pip install twine
+        last_dist=$(ls -t dist/autoPyTorch-*.tar.gz | head -n 1)
+        twine_output=`twine check "$last_dist"`
+        if [[ "$twine_output" != "Checking $last_dist: PASSED" ]]; then echo $twine_output && exit 1;fi
+    - name: Install dist
+      run: |
+        last_dist=$(ls -t dist/autoPyTorch-*.tar.gz | head -n 1)
+        pip install $last_dist
+    - name: PEP 561 Compliance
+      run: |
+        pip install mypy
+        cd ..  # required to use the installed version of autosklearn
+        if ! python -c "import autoPyTorch"; then exit 1; fi

.github/workflows/docs.yml

@@ -0,0 +1,44 @@
+name: Docs
+on: [pull_request, push]
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        git submodule update --init --recursive
+        pip install -e .[docs,examples]
+    - name: Make docs
+      run: |
+        cd docs
+        make html
+    - name: Pull latest gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        cd ..
+        git clone https://github.com/automl/Auto-PyTorch.git --branch gh-pages --single-branch gh-pages
+    - name: Copy new doc into gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        branch_name=${GITHUB_REF##*/}
+        cd ../gh-pages
+        rm -rf $branch_name
+        cp -r ../Auto-PyTorch/docs/build/html $branch_name
+    - name: Push to gh-pages
+      if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
+      run: |
+        last_commit=$(git log --pretty=format:"%an: %s")
+        cd ../gh-pages
+        branch_name=${GITHUB_REF##*/}
+        git add $branch_name/
+        git config --global user.name 'Github Actions'
+        git config --global user.email '[email protected]'
+        git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+        git commit -am "$last_commit"
+        git push

.github/workflows/long_regression_test.yml

@@ -0,0 +1,35 @@
+name: Tests
+
+on:
+  schedule:
+    # Every Truesday at 7AM UTC
+    # TODO teporary set to every day just for the PR
+    #- cron: '0 07 * * 2'
+    - cron: '0 07 * * *'
+
+
+jobs:
+  ubuntu:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+      fail-fast:  false
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: development
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install test dependencies
+      run: |
+        git submodule update --init --recursive
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+    - name: Run tests
+      run: |
+        python -m pytest --durations=200 cicd/test_preselected_configs.py -vs

.github/workflows/pre-commit.yaml

@@ -0,0 +1,23 @@
+name: pre-commit
+
+on: [push, pull_request]
+
+jobs:
+  run-all-files:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Init Submodules
+      run: |
+        git submodule update --init --recursive
+    - name: Install pre-commit
+      run: |
+        pip install pre-commit
+        pre-commit install
+    - name: Run pre-commit
+      run: |
+        pre-commit run --all-files

.github/workflows/pytest.yml

@@ -0,0 +1,55 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        include:
+          - python-version: 3.8
+            code-cov: true
+      fail-fast: false
+      max-parallel: 2
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install test dependencies
+      run: |
+        git submodule update --init --recursive
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+    - name: Store repository status
+      id: status-before
+      run: |
+        echo "::set-output name=BEFORE::$(git status --porcelain -b)"
+    - name: Run tests
+      run: |
+        if [ ${{ matrix.code-cov }} ]; then
+          codecov='--cov=autoPyTorch --cov-report=xml --cov-config=.coveragerc';
+        fi
+        python -m pytest --forked --durations=20 --timeout=600 --timeout-method=signal -v $codecov test
+    - name: Check for files left behind by test
+      if: ${{ always() }}
+      run: |
+        before="${{ steps.status-before.outputs.BEFORE }}"
+        after="$(git status --porcelain -b)"
+        if [[ "$before" != "$after" ]]; then
+            echo "git status from before: $before"
+            echo "git status from after: $after"
+            echo "Not all generated files have been deleted!"
+            exit 1
+        fi
+    - name: Upload coverage
+      if: matrix.code-cov && always()
+      uses: codecov/codecov-action@v1
+      with:
+        fail_ci_if_error: true
+        verbose: true

.github/workflows/scheduled_test.yml

@@ -0,0 +1,34 @@
+name: Tests
+
+on:
+  schedule:
+    # Every Monday at 7AM UTC
+    - cron: '0 07 * * 1'
+
+
+jobs:
+  ubuntu:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+      fail-fast:  false
+      max-parallel: 2
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: development
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install test dependencies
+      run: |
+        git submodule update --init --recursive
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+    - name: Run tests
+      run: |
+        python -m pytest --forked --durations=20 --timeout=600 --timeout-method=signal -v test