Github Actions workflow for running tests

.github/workflows/test.yml

@@ -1,31 +1,94 @@
-name: Test
+name: Tests
 
 # Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
+  # Triggers the workflow on push or pull request events
   push:
-    branches: [ tests ]
   pull_request:
-    branches: [ tests ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-
-  # This workflow contains a single job called "tests"
-  tests:
-    # The type of runner that the job will run on
-    runs-on: ubuntu-latest
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
+  # Running tests on Windows under Windows Subsystem for Linux (WSL)
+  windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [python38, python39]
+        rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2]
+    runs-on: windows-latest
+    name: Windows (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }})
+    defaults:
+      run:
+        shell: wsl-bash {0}
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-
-      - uses: cachix/install-nix-action@v13
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Setup Windows Subsystem for Linux
+        uses: Vampire/setup-wsl@v1
         with:
-          nix_path: nixpkgs=channel:nixos-unstable
+          # Using Ubuntu to setup WSL
+          distribution: Ubuntu-20.04
+          additional-packages:
+            dos2unix
+      - name: Install Nix
+        run: |
+            # Nix installer does not support root installation we need a user
+            useradd test -m
+
+            # Create /nix folder with proper permissions
+            # This is required because otherwise installer gets stuck most likely at running sudo
+            install -d -m755 -o test -g test /nix
+
+            # Download Nix installer
+            sudo -iu test curl -o ./install-nix.sh -L https://nixos.org/nix/install
+
+            # Run installer as single user installation and we do not need a channel
+            sudo -iu test sh ./install-nix.sh --no-daemon --no-channel-add
+      - name: Run tests
+        run: |
+            # Change line endings
+            dos2unix default.nix
+            # Run the tests in project directory where default.nix resides
+            # We need login shell for that so that .profile is getting read
+            sudo -iu test sh -c "nix-build $PWD --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }}"
+            exit $(cat /home/test/result/exitCode)
 
-      - run: nix-shell --argstr cmd "invoke test --in-nix" --argstr python python37 --argstr rfVersion 3.0.4
+  linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [python38, python39]
+        rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2]
+    runs-on: ubuntu-latest
+    name: Linux (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }})
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Install Nix
+        uses: cachix/install-nix-action@v13
+      - name: Run tests
+        run: |
+          nix-build --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }}
+          exit $(cat ./result/exitCode)
+
+  macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [python38, python39]
+        rfVersion: [3.1.2, 3.2, 3.2.1, 3.2.2]
+    runs-on: macos-latest
+    name: MacOS (${{ matrix.python }}, robotframework-${{ matrix.rfVersion }})
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Install Nix
+        uses: cachix/install-nix-action@v13
+      - name: Run tests
+        run: |
+          nix-build --argstr python ${{ matrix.python }} --argstr rfVersion ${{ matrix.rfVersion }}
+          exit $(cat ./result/exitCode)

.gitignore

@@ -13,8 +13,9 @@ dist
 venv
 .coverage
 htmlcov
-green-junit.xml 
+green-junit.xml
 tests/resources/green-junit-example_robot_output.xml
 example/results_robot_output.xml
 example/results.json
 .vscode
+/result

README.md

@@ -109,6 +109,43 @@ and the task file [`tasks.py`](https://github.com/eficode/robotframework-oxygen/
 
 [Read the developer guide on how to write your own handler](DEVGUIDE.md)
 
+# Developing Oxygen with Nix
+
+Nix is being used in this project for building development environments with capability of running bulk tests across multiple python versions and multiple Robot Framework versions.
+
+## Requirements
+
+- Nix (https://nixos.org/download.html#nix-quick-install)
+
+## Development environment
+
+This opens bash shell in current terminal window, with latest python 3.9 and Robot Framework 3.2.2.
+```
+$ nix-shell --argstr python python39 --argstr rfVersion 3.2.2
+```
+Now you can run the tests, for example:
+```
+$ invoke test --in-nix
+$ invoke utest --in-nix
+$ invoke atest
+```
+
+To exit the environment/shell type `<Ctrl+d>` or:
+```
+$ exit
+```
+
+## Bulk tests
+
+This command tests all currently supported combinations of Python and Robot Framework.
+```
+$ nix-build test.nix
+```
+It should run for few minutes, and if all tests pass, the output will be:
+```
+Overall tests state: ok
+```
+
 # License
 
 Details of project licensing can be found in the [LICENSE](LICENSE) file in the project repository.

default.nix

@@ -0,0 +1,122 @@
+/* Nix script used primarily for running tests per one specific Python version
+  and one specific RobotFramework version.
+
+  For running this script Nix has to be installed on system, the rest of the
+  dependencies (Python and other dependencies) will be taken care of by Nix itself.
+
+  Python packages are supplied by Nix while RobotFramework is fetched from PyPi
+
+  Running tests:
+    All arguments are optional, by default latest Python 3.9,
+    RobotFramework 3.2.2 is used and tests are being run with command `invoke test --in-nix`
+
+    Example:
+    $ nix-build --argstr python python39 --argstr rfVersion 3.2.2 --argstr cmd "invoke test --in-nix"
+
+  Secondary function of this script is development environment:
+    User can use nix-shell to drop into development shell with all dependencies needed for the project,
+    without the need to separately install all supported Python versions and/or other dependencies on your own.
+
+    Example:
+    This will drop the user into subshell with latest Python 3.8 and RobotFramework version 3.0.4,
+    ready to run tests or your favorite editor out of it (so it is running in correct environment)
+    $ nix-shell --argstr python python38 --argstr rfVersion 3.0.4 --argstr cmd "$SHELL"
+*/
+{ nixpkgsBranch ? "release-21.05"
+, nixpkgs ? "https://github.com/NixOS/nixpkgs/archive/refs/heads/${nixpkgsBranch}.tar.gz"
+, python ? "python39"
+, rfVersion ? "3.2.2"
+, path ? toString ./.
+, cmd ? "invoke test --in-nix" }:
+let
+  pkgs = import (fetchTarball nixpkgs) { };
+in
+with pkgs;
+with lib;
+let
+  mach-nix = import (builtins.fetchGit {
+    url = "https://github.com/DavHau/mach-nix/";
+    ref = "refs/tags/3.3.0";
+  }) {
+    inherit pkgs python;
+    pypiDataRev = "c8393888d97e74f2217aaafae70bf5bc5c913535";
+    pypiDataSha256 = "0pfivp1w3pdbsamnmmyy4rsnc4klqnmisjzcq0smc4pp908b6sc3";
+  };
+
+  /* Returns new line delimited dependencies in format of `requirements.txt`
+    without the specific dependencies that are passed as list of strings.
+
+    The `requirements.txt` is picked up from root directory of the project.
+
+    Example of requirements.txt content:
+    ```
+    foo==1.2.3
+    bar<2.0
+    baz==2.3.4
+    qux
+    ```
+
+    Example run:
+    result = requirementsWithout ['foo', 'qux']
+
+    Result for this example:
+    ```
+    bar<2.0
+    baz==2.3.4
+    ```
+  */
+  requirementsWithout = names:
+    let
+      requirementsFile = names: runCommand "requirements.txt" {
+        buildInputs = [ gnused ];
+      } ''
+        cp "${/. + "${path}/requirements.txt"}" $out
+        for name in ${toString names}
+        do
+          sed -E "/^$name([=<>]|\$)/d" -i $out
+        done
+      '';
+    in
+      readFile (requirementsFile names);
+
+  pname = "robotframework-oxygen";
+  requirements = ''
+    ${requirementsWithout ["robotframework"]}
+    robotframework==${rfVersion}
+  '';
+
+  env = mach-nix.mkPython {
+    inherit requirements;
+  };
+in
+  runCommand "${pname}-result" {
+    buildInputs = [ which coreutils env ];
+    # Hook for when the file is being run with nix-shell for the devel environment
+    shellHook = ''
+      cd ${path}
+      export PYTHONPATH="$PWD/src:${env}/lib/${env.python.libPrefix}/site-packages"
+      ${cmd}
+      exitCode=$?
+      echo -e "\n[${python}][robotframework==${rfVersion}] Exited with $exitCode"
+      exit $exitCode
+    '';
+  } ''
+    mkdir -p $out
+    tmpdir="$(mktemp -d -t ${pname}-XXXXXXXXXX)"
+    trap "rm -rf $tmpdir" EXIT
+    cp -r ${/. + path} $tmpdir/src
+    chmod -R u+w $tmpdir
+    cd $tmpdir/src
+    export PYTHONPATH="$PWD/src:${env}/lib/${env.python.libPrefix}/site-packages"
+    export HOME=$tmpdir
+    echo '{"run":{"shell":"${stdenv.shell}"}}' >$HOME/.invoke.json
+    exec &> >(tee $out/log)
+    set +e
+    ${cmd}
+    exitCode=$?
+    set -e
+    echo -e "\n[${python}][robotframework==${rfVersion}] Exited with $exitCode"
+    echo "$exitCode" > $out/exitCode
+    echo "${python}" > $out/python
+    echo "${rfVersion}" > $out/rfVersion
+  ''

tasks.py

@@ -7,6 +7,9 @@
 CURDIR = Path.cwd()
 SRCPATH = CURDIR / 'src'
 UNIT_TESTS = CURDIR / 'tests'
+IN_NIX_HELP = ('Setup environment for use inside Nix. By default PYTHONPATH '
+               'is being overriden with source path. This flag makes sure '
+               'that PYTHONPATH is untouched.')
 
 # If you want colored output for the tasks, use `run()` with `pty=True`
 # Not on Windows, though -- it'll fail if you have `pty=True`
@@ -35,34 +38,44 @@ def install(context, package=None):
       help={
           'test': 'Limit unit test execution to specific tests. Must be given '
                   'multiple times to select several targets. See more: '
-                  'https://github.com/CleanCut/green/blob/master/cli-options.txt#L5'
+                  'https://github.com/CleanCut/green/blob/master/cli-options.txt#L5',
+          'in_nix': IN_NIX_HELP
       })
-def utest(context, test=None):
+def utest(context, test=None, in_nix=False):
+    env = {} if in_nix else {'PYTHONPATH': str(SRCPATH)}
     run(f'green {" ".join(test) if test else UNIT_TESTS}',
-        env={'PYTHONPATH': str(SRCPATH)},
+        env=env,
         pty=(not system() == 'Windows'))
-    run('coverage html')
 
-@task
-def coverage(context):
-  run(f'green -r {str(UNIT_TESTS)}',
-      env={'PYTHONPATH': str(SRCPATH)},
-      pty=(not system() == 'Windows'))
+@task(help={
+    'in_nix': IN_NIX_HELP
+})
+def coverage(context, in_nix=False):
+    env = {} if in_nix else {'PYTHONPATH': str(SRCPATH)}
+    run(f'green -r {str(UNIT_TESTS)}',
+        env=env,
+        pty=(not system() == 'Windows'))
+    run('coverage html')
 
-@task(help={'rf': 'Additional command-line arguments for Robot Framework as '
-                  'single string. E.g: invoke atest --rf "--name my_suite"'})
+@task(help={
+    'rf': 'Additional command-line arguments for Robot Framework as '
+          'single string. E.g: invoke atest --rf "--name my_suite"'
+})
 def atest(context, rf=''):
     run(f'robot '
-        f'--pythonpath {str(SRCPATH)} '
+        f'--pythonpath {str(SRCPATH)}'
         f'--dotted '
         f'{rf} '
         f'--listener oxygen.listener '
         f'{str(CURDIR / "tests" / "atest")}',
         pty=(not system() == 'Windows'))
 
-@task(pre=[utest, atest])
-def test(context):
-    pass
+@task(help={
+    'in_nix': IN_NIX_HELP
+})
+def test(context, in_nix=False):
+    utest(context, in_nix=in_nix)
+    atest(context)
 
 @task
 def doc(context):