make: add target to make a code coverage report

Patch introduces integration with coverage.py module [1] that allows to
gather code coverage and build a simple report about covered statements
and branches. We have a test-run.py module itself and also we have Python
helpers for Tarantool tests written in Python. These Python tests runs a
separate processes using subprocess module. To make it possible to gather code
coverage for these helpers there are special changes added to
lib/tarantool_server.py file.

Additionally patch introduces integration with Coveralls
service that allows to evaluate incremental code coverage changes like
we do it for Tarantool [2]. Code coverage for test-run is available on
page by URL [3].

Name                          Stmts   Miss Branch BrPart    Cover
-----------------------------------------------------------------
dispatcher.py                   281     57     94     13    76.0%
lib/__init__.py                  45      3      6      2    90.2%
lib/admin_connection.py          71     50     22      0    22.6%
lib/app_server.py               162    104     38      2    33.0%
lib/box_connection.py            54     33     12      0    31.8%
lib/colorer.py                  111     31     42      7    64.7%
lib/connpool.py                  85     65     18      0    19.4%
lib/inspector.py                 81     49     26      1    34.6%
lib/options.py                   81     15     18      6    78.8%
lib/preprocessor.py             329    290    162      0     7.9%
lib/pytap13.py                  128    110     58      1    10.2%
lib/server.py                   111     62     40      2    35.1%
lib/server_mixins.py            126     83     28      0    27.9%
lib/tarantool_connection.py     126     89     22      0    25.0%
lib/tarantool_server.py         735    553    270      9    20.6%
lib/test.py                     249    181     86      0    20.3%
lib/test_suite.py               191     76     68     12    57.5%
lib/unittest_server.py           63     25     10      2    57.5%
lib/utils.py                    216    132     86      9    33.4%
lib/worker.py                   251    179     70      3    25.2%
listeners.py                    205     78     70      6    60.0%
test-run.py                     149     76     44      9    42.5%
-----------------------------------------------------------------
TOTAL                          3850   2341   1290     84    34.2%

1. https://coverage.readthedocs.io/en/latest/config.html
2. https://coveralls.io/github/tarantool/tarantool
3. https://coveralls.io/github/tarantool/test-run

Author: ligurio

.coveragerc

@@ -0,0 +1,10 @@
+[run]
+branch = True
+
+[report]
+precision = 1
+include =
+    ./*
+omit =
+    lib/tarantool-python/*
+    lib/msgpack-python/*

.github/workflows/test.yml

@@ -58,3 +58,14 @@ jobs:
     - name: run regression tests
       run: |
        make test
+    - name: code coverage
+      run: |
+       make coverage
+    - name: upload coverage data to coveralls.io
+      run: coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+        COVERALLS_PARALLEL: true
+      include:
+        - python-version: 3.8
+        - tarantool-version: 2.7

Makefile

@@ -1,3 +1,5 @@
+MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+PROJECT_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH)))
 TEST_RUN_EXTRA_PARAMS?=
 PYTHON?=python
 
@@ -13,11 +15,18 @@ luacheck:
 	luacheck --config .luacheckrc .
 
 test_integration:
-	$(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
+	PYTHONPATH=$(PROJECT_DIR) $(PYTHON) test/test-run.py --force $(TEST_RUN_EXTRA_PARAMS)
 
 test_unittest:
 	$(PYTHON) -m unittest discover test/unittest/
 
 test: test_unittest test_integration
 
+coverage:
+	PYTHON="$(PYTHON) -m coverage run" make -f $(MAKEFILE_PATH) test
+	coverage report
+
+clean:
+	coverage erase
+
 .PHONY: lint flake8 luacheck test test_integration test_unittest

README.md

@@ -1,5 +1,7 @@
 # Tarantool Functional testing framework
 
+[![Coverage Status](https://coveralls.io/repos/github/tarantool/test-run/badge.svg)](https://coveralls.io/github/tarantool/test-run)
+
 ### Test Suite
 
 Bunch of tests, that lay down in the subfolder (recursively) with `suite.ini`

lib/tarantool_server.py

@@ -857,6 +857,12 @@ def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
 
         # redirect stdout from tarantoolctl and tarantool
         os.putenv("TEST_WORKDIR", self.vardir)
+        if os.environ.get('PYTHON') == 'coverage run':
+            import coverage
+            coveragerc_path = os.path.join(os.path.abspath(self.sourcedir), ".coveragerc")
+            assert(os.path.exists(coveragerc_path))
+            os.environ['COVERAGE_PROCESS_START'] = os.path.join(coveragerc_path)
+            coverage.process_startup()
         self.process = subprocess.Popen(args,
                                         cwd=self.vardir,
                                         stdout=self.log_des,

requirements-test.txt

@@ -1,2 +1,4 @@
+coverage==5.*
+coveralls==1.*
 flake8==3.7.9
 hypothesis==4.*