make: add target to make a code coverage report

Patch introduces integration with coverage.py module [1] that allows to
gather code coverage and build a simple report about covered statements
and branches. We have a test-run.py module itself and also we have Python
helpers for Tarantool tests written in Python. These Python tests runs a
separate processes using subprocess module. To make it possible to gather code
coverage for a separate processes option 'concurrency' added to .coveragerc.

Additionally patch introduces integration with Coveralls
service using python-coveralls module [2] that allows to evaluate
incremental code coverage changes like we do it for Tarantool [3]. Code
coverage for test-run is available on a public page by URL [4].

Name                          Stmts   Miss Branch BrPart    Cover
-----------------------------------------------------------------
dispatcher.py                   281     50     94     12    78.1%
lib/__init__.py                  45      0      6      1    98.0%
lib/admin_connection.py          71      9     22      7    82.8%
lib/app_server.py               162     45     38     11    67.0%
lib/box_connection.py            54     22     12      1    50.0%
lib/colorer.py                  111     28     42      6    68.6%
lib/connpool.py                  85     54     18      1    35.0%
lib/inspector.py                 81     13     26      4    80.4%
lib/options.py                   81     15     18      6    78.8%
lib/preprocessor.py             329    162    162     40    44.6%
lib/pytap13.py                  128     48     58     18    55.9%
lib/server.py                   111     23     40     12    74.2%
lib/server_mixins.py            126     83     28      0    27.9%
lib/tarantool_connection.py     126     28     22      5    75.0%
lib/tarantool_server.py         729    237    268     65    61.5%
lib/test.py                     249    111     86     17    48.1%
lib/test_suite.py               191     35     68     19    77.6%
lib/unittest_server.py           63      6     10      4    86.3%
lib/utils.py                    216     76     86     19    59.9%
lib/worker.py                   251     81     70     15    60.7%
listeners.py                    205     78     70      6    60.0%
test-run.py                     149     76     44      9    42.5%
-----------------------------------------------------------------
TOTAL                          3844   1280   1288    278    61.2%

1. https://coverage.readthedocs.io/en/latest/config.html
2. https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html#github-actions-support
3. https://coveralls.io/github/tarantool/tarantool
4. https://coveralls.io/github/tarantool/test-run

Author: ligurio

.coveragerc

@@ -0,0 +1,14 @@
+[run]
+branch = True
+concurrency =
+    gevent
+    multiprocessing
+
+[report]
+precision = 1
+include =
+    ./*
+omit =
+    lib/tarantool-python/*
+    lib/msgpack-python/*
+    test/*

.github/workflows/test.yml

@@ -58,3 +58,13 @@ jobs:
     - name: run regression tests
       run: |
        make test
+    - name: code coverage
+      if: ${{ matrix.python-version == '3.8' && matrix.tarantool-version == '2.7' }}
+      run: |
+       pip install coveralls==3.*
+       make coverage
+    - name: upload coverage data to coveralls.io
+      if: ${{ matrix.python-version == '3.8' && matrix.tarantool-version == '2.7' }}
+      run: coveralls --service=github
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Makefile

@@ -1,3 +1,5 @@
+MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+PROJECT_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH)))
 TEST_RUN_EXTRA_PARAMS?=
 PYTHON?=python
 
@@ -13,11 +15,19 @@ luacheck:
 	luacheck --config .luacheckrc .
 
 test_integration:
-	$(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
+	PYTHONPATH=$(PROJECT_DIR) $(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
 
 test_unittest:
 	$(PYTHON) -m unittest discover test/unittest/
 
 test: test_unittest test_integration
 
+coverage:
+	PYTHON="coverage run" make -f $(MAKEFILE_PATH) test
+	coverage combine $(PROJECT_DIR) $(PROJECT_DIR)/test
+	coverage report
+
+clean:
+	coverage erase
+
 .PHONY: lint flake8 luacheck test test_integration test_unittest

README.md

@@ -1,5 +1,7 @@
 # Tarantool Functional testing framework
 
+[![Coverage Status](https://coveralls.io/repos/github/tarantool/test-run/badge.svg)](https://coveralls.io/github/tarantool/test-run)
+
 ### Test Suite
 
 Bunch of tests, that lay down in the subfolder (recursively) with `suite.ini`

requirements-test.txt

@@ -1,2 +1,3 @@
+coverage==5.*
 flake8==3.7.9
 hypothesis==4.*