diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c470dd..b6d8a2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,31 +1,94 @@ -name: Test +name: Tests # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the master branch + # Triggers the workflow on push or pull request events push: - branches: [ tests ] pull_request: - branches: [ tests ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - - # This workflow contains a single job called "tests" - tests: - # The type of runner that the job will run on - runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job + # Running tests on Windows under Windows Subsystem for Linux (WSL) + windows: + strategy: + fail-fast: false + matrix: + python: [python38, python39] + rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2] + runs-on: windows-latest + name: Windows (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }}) + defaults: + run: + shell: wsl-bash {0} steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - - uses: cachix/install-nix-action@v13 + - name: Checkout the repository + uses: actions/checkout@v2 + - name: Setup Windows Subsystem for Linux + uses: Vampire/setup-wsl@v1 with: - nix_path: nixpkgs=channel:nixos-unstable + # Using Ubuntu to setup WSL + distribution: Ubuntu-20.04 + additional-packages: + dos2unix + - name: Install Nix + run: | + # Nix installer does not support root installation we need a user + useradd test -m + + # Create /nix folder with proper permissions + # This is required because otherwise installer gets stuck most likely at running sudo + install -d -m755 -o test -g test /nix + + # Download Nix installer + sudo -iu test curl -o ./install-nix.sh -L https://nixos.org/nix/install + + # Run installer as single user installation and we do not need a channel + sudo -iu test sh ./install-nix.sh --no-daemon --no-channel-add + - name: Run tests + run: | + # Change line endings + dos2unix default.nix + # Run the tests in project directory where default.nix resides + # We need login shell for that so that .profile is getting read + sudo -iu test sh -c "nix-build $PWD --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }}" + exit $(cat /home/test/result/exitCode) - - run: nix-shell --argstr cmd "invoke test --in-nix" --argstr python python37 --argstr rfVersion 3.0.4 + linux: + strategy: + fail-fast: false + matrix: + python: [python38, python39] + rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2] + runs-on: ubuntu-latest + name: Linux (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }}) + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + - name: Install Nix + uses: cachix/install-nix-action@v13 + - name: Run tests + run: | + nix-build --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }} + exit $(cat ./result/exitCode) + + macos: + strategy: + fail-fast: false + matrix: + python: [python38, python39] + rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2] + runs-on: macos-latest + name: MacOS (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }}) + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + - name: Install Nix + uses: cachix/install-nix-action@v13 + - name: Run tests + run: | + nix-build --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }} + exit $(cat ./result/exitCode) diff --git a/.gitignore b/.gitignore index df08a01..c4d05df 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,9 @@ dist venv .coverage htmlcov -green-junit.xml +green-junit.xml tests/resources/green-junit-example_robot_output.xml example/results_robot_output.xml example/results.json .vscode +/result diff --git a/README.md b/README.md index 0b38bb9..67bb353 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,43 @@ and the task file [`tasks.py`](https://github.com/eficode/robotframework-oxygen/ [Read the developer guide on how to write your own handler](DEVGUIDE.md) +# Developing Oxygen with Nix + +Nix is being used in this project for building development environments with capability of running bulk tests across multiple python versions and multiple Robot Framework versions. + +## Requirements + +- Nix (https://nixos.org/download.html#nix-quick-install) + +## Development environment + +This opens bash shell in current terminal window, with latest python 3.9 and Robot Framework 3.2.2. +``` +$ nix-shell --argstr python python39 --argstr rfVersion 3.2.2 +``` +Now you can run the tests, for example: +``` +$ invoke test --in-nix +$ invoke utest --in-nix +$ invoke atest +``` + +To exit the environment/shell type `` or: +``` +$ exit +``` + +## Bulk tests + +This command tests all currently supported combinations of Python and Robot Framework. +``` +$ nix-build test.nix +``` +It should run for few minutes, and if all tests pass, the output will be: +``` +Overall tests state: ok +``` + # License Details of project licensing can be found in the [LICENSE](LICENSE) file in the project repository. diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..b4813b2 --- /dev/null +++ b/default.nix @@ -0,0 +1,122 @@ +/* Nix script used primarily for running tests per one specific Python version + and one specific RobotFramework version. + + For running this script Nix has to be installed on system, the rest of the + dependencies (Python and other dependencies) will be taken care of by Nix itself. + + Python packages are supplied by Nix while RobotFramework is fetched from PyPi + + Running tests: + All arguments are optional, by default latest Python 3.9, + RobotFramework 3.2.2 is used and tests are being run with command `invoke test --in-nix` + + Example: + $ nix-build --argstr python python39 --argstr rfVersion 3.2.2 --argstr cmd "invoke test --in-nix" + + Secondary function of this script is development environment: + User can use nix-shell to drop into development shell with all dependencies needed for the project, + without the need to separately install all supported Python versions and/or other dependencies on your own. + + Example: + This will drop the user into subshell with latest Python 3.8 and RobotFramework version 3.0.4, + ready to run tests or your favorite editor out of it (so it is running in correct environment) + $ nix-shell --argstr python python38 --argstr rfVersion 3.0.4 --argstr cmd "$SHELL" +*/ +{ nixpkgsBranch ? "release-21.05" +, nixpkgs ? "https://github.com/NixOS/nixpkgs/archive/refs/heads/${nixpkgsBranch}.tar.gz" +, python ? "python39" +, rfVersion ? "3.2.2" +, path ? toString ./. +, cmd ? "invoke test --in-nix" }: +let + pkgs = import (fetchTarball nixpkgs) { }; +in +with pkgs; +with lib; +let + mach-nix = import (builtins.fetchGit { + url = "https://github.com/DavHau/mach-nix/"; + ref = "refs/tags/3.3.0"; + }) { + inherit pkgs python; + pypiDataRev = "c8393888d97e74f2217aaafae70bf5bc5c913535"; + pypiDataSha256 = "0pfivp1w3pdbsamnmmyy4rsnc4klqnmisjzcq0smc4pp908b6sc3"; + }; + + /* Returns new line delimited dependencies in format of `requirements.txt` + without the specific dependencies that are passed as list of strings. + + The `requirements.txt` is picked up from root directory of the project. + + Example of requirements.txt content: + ``` + foo==1.2.3 + bar<2.0 + baz==2.3.4 + qux + ``` + + Example run: + result = requirementsWithout ['foo', 'qux'] + + Result for this example: + ``` + bar<2.0 + baz==2.3.4 + ``` + */ + requirementsWithout = names: + let + requirementsFile = names: runCommand "requirements.txt" { + buildInputs = [ gnused ]; + } '' + cp "${/. + "${path}/requirements.txt"}" $out + for name in ${toString names} + do + sed -E "/^$name([=<>]|\$)/d" -i $out + done + ''; + in + readFile (requirementsFile names); + + pname = "robotframework-oxygen"; + requirements = '' + ${requirementsWithout ["robotframework"]} + robotframework==${rfVersion} + ''; + + env = mach-nix.mkPython { + inherit requirements; + }; +in + runCommand "${pname}-result" { + buildInputs = [ which coreutils env ]; + # Hook for when the file is being run with nix-shell for the devel environment + shellHook = '' + cd ${path} + export PYTHONPATH="$PWD/src:${env}/lib/${env.python.libPrefix}/site-packages" + ${cmd} + exitCode=$? + echo -e "\n[${python}][robotframework==${rfVersion}] Exited with $exitCode" + exit $exitCode + ''; + } '' + mkdir -p $out + tmpdir="$(mktemp -d -t ${pname}-XXXXXXXXXX)" + trap "rm -rf $tmpdir" EXIT + cp -r ${/. + path} $tmpdir/src + chmod -R u+w $tmpdir + cd $tmpdir/src + export PYTHONPATH="$PWD/src:${env}/lib/${env.python.libPrefix}/site-packages" + export HOME=$tmpdir + echo '{"run":{"shell":"${stdenv.shell}"}}' >$HOME/.invoke.json + exec &> >(tee $out/log) + set +e + ${cmd} + exitCode=$? + set -e + echo -e "\n[${python}][robotframework==${rfVersion}] Exited with $exitCode" + echo "$exitCode" > $out/exitCode + echo "${python}" > $out/python + echo "${rfVersion}" > $out/rfVersion + '' diff --git a/tasks.py b/tasks.py index a400b3c..4ebc07c 100644 --- a/tasks.py +++ b/tasks.py @@ -7,6 +7,9 @@ CURDIR = Path.cwd() SRCPATH = CURDIR / 'src' UNIT_TESTS = CURDIR / 'tests' +IN_NIX_HELP = ('Setup environment for use inside Nix. By default PYTHONPATH ' + 'is being overriden with source path. This flag makes sure ' + 'that PYTHONPATH is untouched.') # If you want colored output for the tasks, use `run()` with `pty=True` # Not on Windows, though -- it'll fail if you have `pty=True` @@ -35,34 +38,44 @@ def install(context, package=None): help={ 'test': 'Limit unit test execution to specific tests. Must be given ' 'multiple times to select several targets. See more: ' - 'https://github.com/CleanCut/green/blob/master/cli-options.txt#L5' + 'https://github.com/CleanCut/green/blob/master/cli-options.txt#L5', + 'in_nix': IN_NIX_HELP }) -def utest(context, test=None): +def utest(context, test=None, in_nix=False): + env = {} if in_nix else {'PYTHONPATH': str(SRCPATH)} run(f'green {" ".join(test) if test else UNIT_TESTS}', - env={'PYTHONPATH': str(SRCPATH)}, + env=env, pty=(not system() == 'Windows')) - run('coverage html') -@task -def coverage(context): - run(f'green -r {str(UNIT_TESTS)}', - env={'PYTHONPATH': str(SRCPATH)}, - pty=(not system() == 'Windows')) +@task(help={ + 'in_nix': IN_NIX_HELP +}) +def coverage(context, in_nix=False): + env = {} if in_nix else {'PYTHONPATH': str(SRCPATH)} + run(f'green -r {str(UNIT_TESTS)}', + env=env, + pty=(not system() == 'Windows')) + run('coverage html') -@task(help={'rf': 'Additional command-line arguments for Robot Framework as ' - 'single string. E.g: invoke atest --rf "--name my_suite"'}) +@task(help={ + 'rf': 'Additional command-line arguments for Robot Framework as ' + 'single string. E.g: invoke atest --rf "--name my_suite"' +}) def atest(context, rf=''): run(f'robot ' - f'--pythonpath {str(SRCPATH)} ' + f'--pythonpath {str(SRCPATH)}' f'--dotted ' f'{rf} ' f'--listener oxygen.listener ' f'{str(CURDIR / "tests" / "atest")}', pty=(not system() == 'Windows')) -@task(pre=[utest, atest]) -def test(context): - pass +@task(help={ + 'in_nix': IN_NIX_HELP +}) +def test(context, in_nix=False): + utest(context, in_nix=in_nix) + atest(context) @task def doc(context): diff --git a/test.nix b/test.nix new file mode 100644 index 0000000..73f197b --- /dev/null +++ b/test.nix @@ -0,0 +1,78 @@ +/* Nix script for running tests with different combinations of Python and + RobotFramework versions in one go. + + Currently this script is not being used by CI/CD but is an option to run + matrix style tests on user's machine. + + Example: + $ nix-build test.nix --argstr pythons "python38 python39" \ + --argstr rfVersions "3.1.2 3.2 3.2.1 3.2.2" + + Example command will run following tests in parallel: + - latest Python 3.8 with RobotFramework 3.1.2 + - latest Python 3.9 with RobotFramework 3.1.2 + - latest Python 3.8 with RobotFramework 3.2 + - latest Python 3.9 with RobotFramework 3.2 + - latest Python 3.8 with RobotFramework 3.2.1 + - latest Python 3.9 with RobotFramework 3.2.1 + - latest Python 3.8 with RobotFramework 3.2.2 + - latest Python 3.9 with RobotFramework 3.2.2 + + Results: + Command will always exit with exit code 0 (unless there is an issue with + the script itself or passing unsupported versions of Python, RobotFramework + or dependencies inside `requirements.txt`). + + The results are in `./result` directory. + + - `./result/state` file will contain value of `ok` or `fail` and is representing the overall state of tests + - `./result///` directory contains files `exitCode` and `log` for each test run +*/ +{ nixpkgsBranch ? "release-21.05" +, nixpkgs ? "https://github.com/NixOS/nixpkgs/archive/refs/heads/${nixpkgsBranch}.tar.gz" +, pythons ? "python38 python39" +, rfVersions ? "3.1.2 3.2 3.2.1 3.2.2" +, path ? toString ./. +, cmd ? "invoke test --in-nix" }: +let + pkgs = import (fetchTarball nixpkgs) { }; +in +with pkgs; +with lib; +let + test = python: rfVersion: + import ./default.nix { inherit nixpkgs python rfVersion path cmd; }; + + pythons' = splitString " " pythons; + rfVersions' = splitString " " rfVersions; + + tests = flatten (map (rfVersion: map (python: test python rfVersion) pythons') rfVersions'); +in + runCommand "test-results" { + buildInputs = [ coreutils gnused ]; + } ('' + mkdir -p $out + echo "python,rfVersion,state,log" >>$out/results.csv + globalState="ok" + '' + (concatMapStringsSep "\n" (test: '' + python="$(cat ${test}/python)" + rfVersion="$(cat ${test}/rfVersion)" + if [ "0" = "$(cat ${test}/exitCode)" ] + then + state="ok" + else + state="fail" + globalState="fail" + fi + testpath="$out/$python/$rfVersion" + mkdir -p "$testpath" + cp "${test}/exitCode" "$testpath/" + cp "${test}/log" "$testpath/" + echo "$python,$rfVersion,$state,$testpath/log" >>$out/results.csv + '') tests) + '' + + echo "$globalState" >$out/state + + sed "s/,/\t/g" <(tail -n +2 "$out/results.csv") + echo -e "\nOverall tests state: $globalState" + '')