From abdd741bd6070fb536426a85ffed7c0f277d14a6 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Mon, 29 Jun 2020 10:24:43 +0300 Subject: [PATCH 01/36] add link to developer guide --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c430c43..03e3cf5 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ $ invoke --list and the task file [`tasks.py`](https://github.com/eficode/robotframework-oxygen/blob/master/tasks.py). +[Read the developer guide on how to write your own handler](DEVGUIDE) + # License Details of project licensing can be found in the [LICENSE](LICENSE) file in the project repository. From 96e4754b93a676b0f558017ddf5369b85a3054d2 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Mon, 29 Jun 2020 10:24:51 +0300 Subject: [PATCH 02/36] add DEVGUIDE --- DEVGUIDE.md | 407 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 DEVGUIDE.md diff --git a/DEVGUIDE.md b/DEVGUIDE.md new file mode 100644 index 0000000..7d3f855 --- /dev/null +++ b/DEVGUIDE.md @@ -0,0 +1,407 @@ +## Oxygen developer guide + +This is a developer guide for Oxygen. We will write a handler for [https://locust.io/](https://locust.io/), which is a performance testing tool. + +# Getting started + +## Prerequisisites + +Python 3 + + +## Start developing + +Let's create a virtual environment and install oxygen. + + +``` +python3 -m venv locustenv +source locustenv/bin/activate +``` + +Install Oxygen by running the following: +``` +$ pip install robotframework-oxygen +``` + +Let's start developing by creating a working folder +```` +cd locustenv +mkdir locust +cd locust +```` + + +### Writing LocustHandler and unit tests + +Let's create `__init__.py` to our `/locust` folder. Next we can write `locusthandler.py` with following content: + +``` +import json +import csv + +from oxygen import BaseHandler +from robot.api import logger + +from oxygen.errors import SubprocessException, LocustHandlerException +from oxygen.utils import run_command_line, validate_path + + +class LocustHandler(BaseHandler): + + + def run_locust(self, result_file, command, check_return_code=False, **env): + '''Run Locust performance testing tool with command + ``command``. + + See documentation for other arguments in \`Run Gatling\`. + ''' + try: + output = run_command_line(command, check_return_code, **env) + except SubprocessException as e: + raise LocustHandlerException(e) + logger.info(output) + logger.info('Result file: {}'.format(result_file)) + return result_file + + def parse_results(self, result_file): + return self._transform_tests(validate_path(result_file).resolve()) + + + def _transform_tests(self, file): + with open(file, newline='') as csvfile: + reader = csv.DictReader(csvfile) + test_cases = [] + for row in reader: + failure_count = row['Failure Count'] + success = failure_count == '0' + keyword = { + 'name': " | ".join(row), + 'pass': success, + 'tags': [], + 'messages': [], + 'teardown': [], + 'keywords': [], + } + test_case = { + 'name': 'Locust test case', + 'tags': [], + 'setup': [], + 'teardown': [], + 'keywords': [keyword] + } + test_cases.append(test_case) + test_suite = { + 'name': 'Locust Scenario', + 'tags': self._tags, + 'setup': [], + 'teardown': [], + 'suites': [], + 'tests': test_cases, + } + return test_suite +``` + +Next we will add LocustHandlerException to `lib/python3.7/site-packages/oxygen/errors.py`, write this to the end of the file: +``` +class LocustHandlerException(Exception): + pass +``` + + We still need to write test file `locust/test_locust.py` with following content: + +``` +from unittest import TestCase + +from pathlib import Path +from locusthandler import LocustHandler + +class TestLocust(TestCase): + + def setUp(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} + self.handler = LocustHandler(config) + path = Path.cwd() / 'requests.csv' + self.test_suite = self.handler.parse_results(path) + + def test_suite_has_four_cases(self): + self.assertEqual(len(self.test_suite['tests']),4) + + def test_pass_is_true_when_failure_request_is_zero(self): + self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) + + def test_pass_is_false_when_failure_request_is_not_zero(self): + self.assertEqual(self.test_suite['tests'][1]['keywords'][0]['pass'], False) +``` + + +and create test data file `locust/requests.csv` which has the following: +``` +"Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" +"GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 +"POST","/",5,5,300,323,288,402,157,0.13,0.13,300,330,330,400,400,400,400,400,400,400,400,400 +"GET","/item",24,0,80,79,67,100,2175,0.63,0.00,81,85,86,86,89,92,100,100,100,100,100,100 +"None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 +``` + +Now we can run unit tests with command + +```` +python -m unittest test_locust +```` + +and all 3 tests should pass. + + +### Configuring LocustHandler to Oxygen + +Let's open the python interpreter from the `locustenv/` directory and check that we can import the locusthandler: + +``` +python +import locust.locusthandler +``` + +running this should not produce any errors, and we can import file `locusthandler.py` from `/locust` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: + +``` +locust.locusthandler: + handler: LocustHandler + keyword: run_locust + tags: oxygen-locusthandler +``` + +### Install demoapp to run tests against + +[Install and run demo-app](https://github.com/robotframework/WebDemo) + +### Running Locust with LocustHandler in Robot test + +First we install locust to our virtualenv: + +``` +pip install locust +``` + +Then we add `locust/locustfile.py` file to `locust` folder which contains the commands for the performance test: + +``` +from locust import HttpUser, task, between + +class QuickstartUser(HttpUser): + wait_time = between(5000, 15000) + + @task + def index_page(self): + self.client.get("/") + +``` + + + +Let's write `locust/test.robot` file which contains test case that runs locust from command line: + +``` +*** Settings *** +Library oxygen.OxygenLibrary +Library OperatingSystem + +*** Variables *** +${STATS_FILE} ${CURDIR}/../example_stats.csv +${FAILURE_FILE} ${CURDIR}/../example_failures.csv +${HISTORY_FILE} ${CURDIR}/../example_stats_history.csv +${LOCUSTFILEPATH} ${CURDIR}/locustfile.py +${LOCUSTCOMMAND} locust -f ${LOCUSTFILEPATH} --headless --host http://localhost:7272 -u 5 -r 1 --run-time 1m --csv=example + +*** Keywords *** + +Remove csv file + [Arguments] ${path} + Remove file ${path} + File should not exist ${path} + +Remove csv files + Remove csv file ${STATS_FILE} + Remove csv file ${FAILURE_FILE} + Remove csv file ${HISTORY_FILE} + + +*** Test Cases *** + +My Locust test + [Tags] LOCUST_ROBOT_TAG + Remove csv files + run_locust + ... ${STATS_FILE} + ... ${LOCUSTCOMMAND} +``` + + + We can run the test by using command + +``` +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +``` + +The test should execute for about 60 seconds. After this you can see the statistics of the performance tests in `log.html` and `report.html`. + + +If the test case fails, you can set variable `check_return_code` to "True" in order to get more specific logging: + +``` +*** Test Cases *** + +My Locust test + [Tags] LOCUST_ROBOT_TAG + Remove csv files + run_locust + ... ${STATS_FILE} + ... ${LOCUSTCOMMAND} + ... check_return_code=${True} +``` + + +## Defining your own parameters + +In our first solution the Locust test case will fail if even one request fails during the performance testing. However this might not be the best way to determine was the performance test successfull or not. Let's implement a solution, where you can define `failure_percentage` , which is the highest percentage of failed request that is allowed in order that the test still passes. + +Let's define define the value of `failure_percentage` in `/lib/python3.7/site-packages/oxygen/config.yml`: + +``` +locust.locusthandler: + handler: LocustHandler + keyword: run_locust + tags: oxygen-locusthandler + failure_percentage: 10 +``` + + +Let's implement function, which returns the failure_percentage to `locust/locusthandler.py`: + +``` + def _get_treshold_failure_percentage(self): + failure_percentage = self._config.get('failure_percentage', None) + + if failure_percentage is None: + print('No failure percentage configured, defaulting to 10') + return 10 + + failure_percentage = int(failure_percentage) + + if failure_percentage > 100: + print('Failure percentage is configured too high, maximizing at 100') + return 100 + + return failure_percentage +``` +and let's use it in `_transform_tests` function: + +``` + def _transform_tests(self, file): + with open(file, newline='') as csvfile: + reader = csv.DictReader(csvfile) + test_cases = [] + for row in reader: + failure_count = row['Failure Count'] + request_count = row['Request Count'] + failure_percentage = 100 * int(failure_count) / int(request_count) + success = failure_percentage <= self._get_treshold_failure_percentage() + keyword = { + 'name': " | ".join(row), + 'pass': success, + 'tags': [], + 'messages': [], + 'teardown': [], + 'keywords': [], + } + test_case = { + 'name': 'Locust test case', + 'tags': [], + 'setup': [], + 'teardown': [], + 'keywords': [keyword] + } + test_cases.append(test_case) + test_suite = { + 'name': 'Locust Scenario', + 'tags': self._tags, + 'setup': [], + 'teardown': [], + 'suites': [], + 'tests': test_cases, + } + return test_suite +``` + +Let's update the tests to match the current functionality. Let's start by defining new data set in `locust/requests.csv`: + +``` +"Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" +"GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 +"POST","/",5,5,300,323,288,402,157,0.13,0.13,300,330,330,400,400,400,400,400,400,400,400,400 +"GET","/item",30,3,80,79,67,100,2175,0.63,0.00,81,85,86,86,89,92,100,100,100,100,100,100 +"None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 +``` + +now we can update the unit tests in `locust/test_locust.py`: + +``` +from unittest import TestCase + +from pathlib import Path +from locusthandler import LocustHandler + +class TestLocust(TestCase): + + def setUp(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} + self.handler = LocustHandler(config) + path = Path.cwd() / 'requests.csv' + self.test_suite = self.handler.parse_results(path) + + def test_suite_has_four_cases(self): + self.assertEqual(len(self.test_suite['tests']),4) + + def test_pass_is_true_when_failure_request_percentage_is_zero(self): + self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) + + def test_pass_is_true_when_failure_request_percentage_is_ten(self): + self.assertEqual(self.test_suite['tests'][2]['keywords'][0]['pass'], True) + + def test_pass_is_false_when_failure_request_percentage_is_above_ten(self): + self.assertEqual(self.test_suite['tests'][1]['keywords'][0]['pass'], False) + + def test_failure_percentage_is_ten_by_default(self): + failure_percentage = self.handler._get_treshold_failure_percentage() + self.assertEqual(failure_percentage, 10) + + def test_failure_percentage_max_amount_is_one_hundred(self): + config = config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '101'} + self.handler = LocustHandler(config) + failure_percentage = self.handler._get_treshold_failure_percentage() + self.assertEqual(failure_percentage, 100) +``` + +Now the unit tests should pass: +```` +python -m unittest test_locust +```` + +And we can also run the robot tests using the new .yaml configuration: + + + +## How to package your project + +[How to package python project](https://packaging.python.org/) + + + +# Teardown + +If you wish to delete your virtual environment do following: + +``` +deactivate +rm -rf locustenv +``` \ No newline at end of file From 8157b985357d622718cddd06314c1735e5fc561c Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Mon, 29 Jun 2020 10:31:31 +0300 Subject: [PATCH 03/36] fix link to dev guide --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 03e3cf5..0b38bb9 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ $ invoke --list and the task file [`tasks.py`](https://github.com/eficode/robotframework-oxygen/blob/master/tasks.py). -[Read the developer guide on how to write your own handler](DEVGUIDE) +[Read the developer guide on how to write your own handler](DEVGUIDE.md) # License From 35ca38dddf1761072630e7a7a64d7b995510ed85 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Mon, 29 Jun 2020 12:00:15 +0300 Subject: [PATCH 04/36] move unit tests of the handler to tests folder --- DEVGUIDE.md | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 7d3f855..cfc126c 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -34,7 +34,7 @@ cd locust ### Writing LocustHandler and unit tests -Let's create `__init__.py` to our `/locust` folder. Next we can write `locusthandler.py` with following content: +Let's create `__init__.py` to our `locustenv/locust` folder. Next we can write `locusthandler.py` with following content: ``` import json @@ -108,7 +108,8 @@ class LocustHandlerException(Exception): pass ``` - We still need to write test file `locust/test_locust.py` with following content: + + Let's create a `tests` folder in `locustenv/locust` . Then we write test file `test_locust.py` with following content: ``` from unittest import TestCase @@ -121,7 +122,7 @@ class TestLocust(TestCase): def setUp(self): config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} self.handler = LocustHandler(config) - path = Path.cwd() / 'requests.csv' + path = Path.cwd() / 'resources/requests.csv' self.test_suite = self.handler.parse_results(path) def test_suite_has_four_cases(self): @@ -135,7 +136,7 @@ class TestLocust(TestCase): ``` -and create test data file `locust/requests.csv` which has the following: +and create test data file `locustenv/locust/resources/requests.csv` which has the following: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" "GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 @@ -144,10 +145,10 @@ and create test data file `locust/requests.csv` which has the following: "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -Now we can run unit tests with command +Now we can run unit tests from the `locustenv/locust` folder with command ```` -python -m unittest test_locust +python -m unittest tests/test_locust.py ```` and all 3 tests should pass. @@ -155,7 +156,7 @@ and all 3 tests should pass. ### Configuring LocustHandler to Oxygen -Let's open the python interpreter from the `locustenv/` directory and check that we can import the locusthandler: +Let's open the python interpreter from the `locustenv` directory and check that we can import the locusthandler: ``` python @@ -183,7 +184,7 @@ First we install locust to our virtualenv: pip install locust ``` -Then we add `locust/locustfile.py` file to `locust` folder which contains the commands for the performance test: +Then we add `locustfile.py` file to `locustenv/locust` folder which contains the commands for the performance test: ``` from locust import HttpUser, task, between @@ -199,7 +200,7 @@ class QuickstartUser(HttpUser): -Let's write `locust/test.robot` file which contains test case that runs locust from command line: +Let's write `locustenv/locust/test.robot` file which contains test case that runs locust from command line: ``` *** Settings *** @@ -237,7 +238,7 @@ My Locust test ``` - We can run the test by using command + We can run the test from `locustenv` folder by using command ``` robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot @@ -276,7 +277,7 @@ locust.locusthandler: ``` -Let's implement function, which returns the failure_percentage to `locust/locusthandler.py`: +Let's implement function, which returns the failure_percentage to `locustenv/locust/locusthandler.py`: ``` def _get_treshold_failure_percentage(self): @@ -333,7 +334,7 @@ and let's use it in `_transform_tests` function: return test_suite ``` -Let's update the tests to match the current functionality. Let's start by defining new data set in `locust/requests.csv`: +Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locust/resources/requests.csv`: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" @@ -343,7 +344,7 @@ Let's update the tests to match the current functionality. Let's start by defini "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -now we can update the unit tests in `locust/test_locust.py`: +now we can update the unit tests in `locust/tests/test_locust.py`: ``` from unittest import TestCase @@ -356,7 +357,7 @@ class TestLocust(TestCase): def setUp(self): config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} self.handler = LocustHandler(config) - path = Path.cwd() / 'requests.csv' + path = Path.cwd() / '/resources/requests.csv' self.test_suite = self.handler.parse_results(path) def test_suite_has_four_cases(self): @@ -382,12 +383,17 @@ class TestLocust(TestCase): self.assertEqual(failure_percentage, 100) ``` -Now the unit tests should pass: +Now the unit tests should pass, run the tests from `locustenv/locust` folder with command: + ```` -python -m unittest test_locust +python -m unittest tests/test_locust.py ```` -And we can also run the robot tests using the new .yaml configuration: +And we can also try out the robot tests using the new .yaml configuration. Run the tests from `locustenv` folder by using command + +``` +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +``` From d7cff9c314031e6ea806581f703e6a775636c93f Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Mon, 29 Jun 2020 13:46:48 +0300 Subject: [PATCH 05/36] add instructions on how to package project --- DEVGUIDE.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index cfc126c..51ff09c 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -399,7 +399,9 @@ robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust ## How to package your project -[How to package python project](https://packaging.python.org/) +Let's package our project in the same virtual environment . Add necessary files to `locustenv/locust` folder and follow the packaging steps defined in this [tutorial](https://packaging.python.org/). + +NOTE: You can use `pip install` instead of `python pip install` because you are already inside active venv. @@ -410,4 +412,6 @@ If you wish to delete your virtual environment do following: ``` deactivate rm -rf locustenv -``` \ No newline at end of file +``` + +And shutdown the demo-app which was tested by locust with CTRL+D \ No newline at end of file From 68e6aca1249f1a28b880ec2c8b6bc3cdb4f136c8 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 30 Jun 2020 11:48:53 +0300 Subject: [PATCH 06/36] fix typo prerequisites --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 51ff09c..cb09ae9 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -4,7 +4,7 @@ This is a developer guide for Oxygen. We will write a handler for [https://locus # Getting started -## Prerequisisites +## Prerequisites Python 3 From 493ff349c9f5ebed40ce292cef15b741a81240a5 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 30 Jun 2020 12:22:44 +0300 Subject: [PATCH 07/36] define LocustHandlerException in locusthandler file --- DEVGUIDE.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index cb09ae9..baa8ed7 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -43,7 +43,7 @@ import csv from oxygen import BaseHandler from robot.api import logger -from oxygen.errors import SubprocessException, LocustHandlerException +from oxygen.errors import SubprocessException from oxygen.utils import run_command_line, validate_path @@ -100,15 +100,11 @@ class LocustHandler(BaseHandler): 'tests': test_cases, } return test_suite -``` - -Next we will add LocustHandlerException to `lib/python3.7/site-packages/oxygen/errors.py`, write this to the end of the file: -``` + class LocustHandlerException(Exception): pass ``` - Let's create a `tests` folder in `locustenv/locust` . Then we write test file `test_locust.py` with following content: ``` @@ -136,7 +132,7 @@ class TestLocust(TestCase): ``` -and create test data file `locustenv/locust/resources/requests.csv` which has the following: +next we create `resources` folder in `locustenv/locust` and add there test data file `requests.csv` which has the following: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" "GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 @@ -176,9 +172,19 @@ locust.locusthandler: [Install and run demo-app](https://github.com/robotframework/WebDemo) +Open up another terminal and run following commands: + + +``` +git clone https://github.com/robotframework/WebDemo.git +cd WebDemo +pip install -r requirements.txt +python demoapp/server.py +``` + ### Running Locust with LocustHandler in Robot test -First we install locust to our virtualenv: +First we install locust to our locustenv virtualenv: ``` pip install locust @@ -357,7 +363,7 @@ class TestLocust(TestCase): def setUp(self): config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} self.handler = LocustHandler(config) - path = Path.cwd() / '/resources/requests.csv' + path = Path.cwd() / 'resources/requests.csv' self.test_suite = self.handler.parse_results(path) def test_suite_has_four_cases(self): From e7c45b6cd681b0641dc2f9a46350fe5a71aa8bce Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 30 Jun 2020 12:57:49 +0300 Subject: [PATCH 08/36] add venv instructions on running WebDemo --- DEVGUIDE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index baa8ed7..bff89c8 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -100,7 +100,7 @@ class LocustHandler(BaseHandler): 'tests': test_cases, } return test_suite - + class LocustHandlerException(Exception): pass ``` @@ -170,7 +170,7 @@ locust.locusthandler: ### Install demoapp to run tests against -[Install and run demo-app](https://github.com/robotframework/WebDemo) +[Next we we install and run demo-app that we run the locust tests against.](https://github.com/robotframework/WebDemo) Open up another terminal and run following commands: @@ -178,6 +178,9 @@ Open up another terminal and run following commands: ``` git clone https://github.com/robotframework/WebDemo.git cd WebDemo +python3 -m venv myenv +source myenv/bin/activate + pip install -r requirements.txt python demoapp/server.py ``` From 99f4a9817d2856687f4e269048d3a2defb91e2b1 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 1 Jul 2020 12:40:01 +0300 Subject: [PATCH 09/36] modify teardown --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index bff89c8..88ec3cc 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -423,4 +423,4 @@ deactivate rm -rf locustenv ``` -And shutdown the demo-app which was tested by locust with CTRL+D \ No newline at end of file +And shutdown the demo-app which was tested by locust with CTRL+D. You can also deactivate and delete `myenv` virtual environment if you wish. \ No newline at end of file From 25771aa228e9c4a4ceaba0e0a2e8cf6e5d4e6aa8 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 1 Jul 2020 14:26:21 +0300 Subject: [PATCH 10/36] add instruction how to add failure percentage as an robot test parameter --- DEVGUIDE.md | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 88ec3cc..973e021 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -404,6 +404,166 @@ And we can also try out the robot tests using the new .yaml configuration. Run t robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot ``` +## Adding failure percentage as an robot test parameter + +Our current solution works quite nicely, but let's imagine a situation where we run the performance tests on different parts of the software, where we wan't to determine different values for the amount of `failure_percentage`. + +Let's change the functionality of `locust/locusthandler.py`: + +``` +import json +import csv + +from oxygen import BaseHandler +from robot.api import logger + +from oxygen.errors import SubprocessException +from oxygen.utils import run_command_line, validate_path + + +class LocustHandler(BaseHandler): + + def run_locust(self, result_file, command, check_return_code=False, failure_percentage=None, **env): + '''Run Locust performance testing tool with command + ``command``. + + See documentation for other arguments in \`Run Gatling\`. + ''' + try: + output = run_command_line(command, check_return_code, **env) + except SubprocessException as e: + raise LocustHandlerException(e) + logger.info(output) + logger.info('Result file: {}'.format(result_file)) + dictionary = dict() + dictionary['result_file'] = result_file + dictionary['failure_percentage'] = failure_percentage + return dictionary + + def parse_results(self, dictionary): + result_file = dictionary['result_file'] + failure_percentage = dictionary['failure_percentage'] + if failure_percentage is None: + failure_percentage = self._config.get('failure_percentage', None) + treshold_failure_percentage = self._get_treshold_failure_percentage(failure_percentage) + print('treshold is: {}'.format(treshold_failure_percentage)) + return self._transform_tests(validate_path(result_file).resolve(), treshold_failure_percentage) + + def _transform_tests(self, file, treshold_failure_percentage): + with open(file, newline='') as csvfile: + reader = csv.DictReader(csvfile) + test_cases = [] + for row in reader: + failure_count = row['Failure Count'] + request_count = row['Request Count'] + failure_percentage = 100 * int(failure_count) / int(request_count) + success = failure_percentage <= treshold_failure_percentage + keyword = { + 'name': " | ".join(row), + 'pass': success, + 'tags': [], + 'messages': [], + 'teardown': [], + 'keywords': [], + } + test_case = { + 'name': 'Locust test case', + 'tags': [], + 'setup': [], + 'teardown': [], + 'keywords': [keyword] + } + test_cases.append(test_case) + test_suite = { + 'name': 'Locust Scenario', + 'tags': self._tags, + 'setup': [], + 'teardown': [], + 'suites': [], + 'tests': test_cases, + } + return test_suite + + def _get_treshold_failure_percentage(self, failure_percentage): + if failure_percentage is None: + print('No failure percentage configured, defaulting to 10') + return 10 + + failure_percentage = int(failure_percentage) + + if failure_percentage > 100: + print('Failure percentage is configured too high, maximizing at 100') + return 100 + + return failure_percentage + +class LocustHandlerException(Exception): + pass +``` + +Notice that we return an dictionary object instead of result file in the `run_locust` method. This way we can use the `failure_percentage` value if it is defined. If it's not defined we will use the value what is defined in `/lib/python3.7/site-packages/oxygen/config.yml`. Now we can rewrite the robot tests in `locust/test.robot`, one assigns the value from the parameter and the second test doesn't: + +``` +*** Test Cases *** + +Critical test + [Tags] LOCUST_ROBOT_TAG + Remove csv files + run_locust + ... ${STATS_FILE} + ... ${LOCUSTCOMMAND} + ... failure_percentage=${1} + + +Normal test + [Tags] LOCUST_ROBOT_TAG + Remove csv files + run_locust + ... ${STATS_FILE} + ... ${LOCUSTCOMMAND} +``` + +In this case the `Critical test` could be a performance test for a system where the consequences of failure is much larger: Thus we define the failure_percentage to 1%. In the `Normal test` we use the value that is defined in the `/lib/python3.7/site-packages/oxygen/config.yml`. + +Run the tests in `locustenv/` folder with + +``` +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +``` + +However now when you run the unit tests from `locust/` folder they fail: + +``` +python -m unittest tests/test_locust.py +``` + +because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup: + +``` + def setUp(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} + self.handler = LocustHandler(config) + path = Path.cwd() / 'resources/requests.csv' + dictionary = dict() + dictionary['result_file'] = path + dictionary['failure_percentage'] = None + self.test_suite = self.handler.parse_results(dictionary) +``` + +and run tests again. Still two test cases fail. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the test cases: + +``` + def test_failure_percentage_is_ten_by_default(self): + failure_percentage = self.handler._get_treshold_failure_percentage(None) + self.assertEqual(failure_percentage, 10) + + def test_failure_percentage_max_amount_is_one_hundred(self): + failure_percentage = self.handler._get_treshold_failure_percentage(101) + self.assertEqual(failure_percentage, 100) +``` + +Now all tests should pass. + ## How to package your project From c8566aef6361b23cbfdb4ab47a2b3b25cfdf947d Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 2 Jul 2020 13:31:09 +0300 Subject: [PATCH 11/36] add more unit tests, add more explanation on functionality --- DEVGUIDE.md | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 973e021..8017694 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -105,6 +105,8 @@ class LocustHandlerException(Exception): pass ``` + Method `run_locust` can be used from robot tests, and it executes the command which runs the locust tests. It returns a path to the locust test results, which is processed by method `parse_results`, which calls `_transform_tests` function which purpose is to transfer the locust result file into a format which can be seen in the Robot Framework result files. + Let's create a `tests` folder in `locustenv/locust` . Then we write test file `test_locust.py` with following content: ``` @@ -333,7 +335,7 @@ and let's use it in `_transform_tests` function: } test_cases.append(test_case) test_suite = { - 'name': 'Locust Scenario', + 'name': 'Locust test suite, failure percentage {}'.form(treshold_failure_percentage), 'tags': self._tags, 'setup': [], 'teardown': [], @@ -475,7 +477,7 @@ class LocustHandler(BaseHandler): } test_cases.append(test_case) test_suite = { - 'name': 'Locust Scenario', + 'name': 'Locust test suite, failure percentage {}'.forma(treshold_failure_percentage), 'tags': self._tags, 'setup': [], 'teardown': [], @@ -537,15 +539,19 @@ However now when you run the unit tests from `locust/` folder they fail: python -m unittest tests/test_locust.py ``` -because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup: +because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup and write a method `dictionary_with_result_file`: ``` - def setUp(self): - config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} - self.handler = LocustHandler(config) + def dictionary_with_result_file(self): path = Path.cwd() / 'resources/requests.csv' dictionary = dict() dictionary['result_file'] = path + return dictionary + + def setUp(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} + self.handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() dictionary['failure_percentage'] = None self.test_suite = self.handler.parse_results(dictionary) ``` @@ -562,7 +568,27 @@ and run tests again. Still two test cases fail. This is because the `_get_tresho self.assertEqual(failure_percentage, 100) ``` -Now all tests should pass. +Now all tests should pass. Let's add two more test cases to see that the `failure_precentage` is set correctly from the parameter or config file. + +``` + def test_parse_results_takes_failure_percentage_from_parameter_prior_to_config(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '70'} + self.handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() + dictionary['failure_percentage'] = 75 + test_suite = self.handler.parse_results(dictionary) + self.assertEqual(test_suite['name'], 'Locust test suite, failure percentage 75') + + def test_parse_results_takes_failure_percentage_correctly_from_config(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '70'} + self.handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() + dictionary['failure_percentage'] = None + test_suite = self.handler.parse_results(dictionary) + self.assertEqual(test_suite['name'], 'Locust test suite, failure percentage 70') +``` + +All 8 tests should pass. Now we have completed an LocustHandler with unit tests which test the most important functionalities. From a197b6ab72fb8120463d30e868c63fcdf8f631c0 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 2 Jul 2020 13:51:19 +0300 Subject: [PATCH 12/36] fix typos --- DEVGUIDE.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 8017694..bf6edf0 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -284,7 +284,7 @@ locust.locusthandler: handler: LocustHandler keyword: run_locust tags: oxygen-locusthandler - failure_percentage: 10 + failure_percentage: 20 ``` @@ -316,8 +316,8 @@ and let's use it in `_transform_tests` function: for row in reader: failure_count = row['Failure Count'] request_count = row['Request Count'] - failure_percentage = 100 * int(failure_count) / int(request_count) - success = failure_percentage <= self._get_treshold_failure_percentage() + treshold_failure_percentage = self._get_treshold_failure_percentage() + success = failure_percentage <= treshold_failure_percentage keyword = { 'name': " | ".join(row), 'pass': success, @@ -335,7 +335,7 @@ and let's use it in `_transform_tests` function: } test_cases.append(test_case) test_suite = { - 'name': 'Locust test suite, failure percentage {}'.form(treshold_failure_percentage), + 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), 'tags': self._tags, 'setup': [], 'teardown': [], @@ -448,7 +448,6 @@ class LocustHandler(BaseHandler): if failure_percentage is None: failure_percentage = self._config.get('failure_percentage', None) treshold_failure_percentage = self._get_treshold_failure_percentage(failure_percentage) - print('treshold is: {}'.format(treshold_failure_percentage)) return self._transform_tests(validate_path(result_file).resolve(), treshold_failure_percentage) def _transform_tests(self, file, treshold_failure_percentage): @@ -477,7 +476,7 @@ class LocustHandler(BaseHandler): } test_cases.append(test_case) test_suite = { - 'name': 'Locust test suite, failure percentage {}'.forma(treshold_failure_percentage), + 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), 'tags': self._tags, 'setup': [], 'teardown': [], @@ -556,7 +555,7 @@ because we changed the functionality to use dictionary instead of result file pa self.test_suite = self.handler.parse_results(dictionary) ``` -and run tests again. Still two test cases fail. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the test cases: +and run tests again. Still two test cases fail. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the failing test cases: ``` def test_failure_percentage_is_ten_by_default(self): From b140158632db937381cf3d2685cd68bffb033644 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 7 Jul 2020 13:05:47 +0300 Subject: [PATCH 13/36] Add instructions on packaging project and verifying that the installation is successful --- DEVGUIDE.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index bf6edf0..37c30a0 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -277,7 +277,7 @@ My Locust test In our first solution the Locust test case will fail if even one request fails during the performance testing. However this might not be the best way to determine was the performance test successfull or not. Let's implement a solution, where you can define `failure_percentage` , which is the highest percentage of failed request that is allowed in order that the test still passes. -Let's define define the value of `failure_percentage` in `/lib/python3.7/site-packages/oxygen/config.yml`: +Let's define the value of `failure_percentage` in `/lib/python3.7/site-packages/oxygen/config.yml`: ``` locust.locusthandler: @@ -593,10 +593,62 @@ All 8 tests should pass. Now we have completed an LocustHandler with unit tests ## How to package your project -Let's package our project in the same virtual environment . Add necessary files to `locustenv/locust` folder and follow the packaging steps defined in this [tutorial](https://packaging.python.org/). +Let's package our project in the same virtual environment . Add necessary files defined in this [tutorial](https://packaging.python.org/) to `locustenv` folder and install `setuptools` and `wheel` to your virtualenv with `pip install`. Add following line yo your `setup.py` file in the `setuptools.setup() object`: -NOTE: You can use `pip install` instead of `python pip install` because you are already inside active venv. +``` + install_requires=[ + 'robotframework-oxygen>=0.1', + 'locust>=1.1', + ], +``` + +so that oxygen including it's dependencies and locust will be installed when your handler is installed. Next you can run following command from `locustenv` folder: + +``` +python setup.py sdist bdist_wheel +``` + +which will create you a `locustenv/dist` folder. Next we will ensure that the installation works by creating another virtualenv. Open up another terminal, go backwards with `cd ..` same path where `locustenv` is and run following commands: + +``` +python3 -m venv packagenv +source packagenv/bin/activate +pip install locustenv/dist/NAME-OF-YOUR-PACKAGE.whl +``` + +you should now have a version of locusthandler in your `packagenv` environment. Let's verify this by opening python intepreter: + +``` +python +import locust.locusthandler +``` + +which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: + +``` +locust.locusthandler: + handler: LocustHandler + keyword: run_locust + tags: oxygen-locusthandler + failure_percentage: 20 +``` + + + Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder , make the change above to the variables in `test.robot`: + ``` +*** Variables *** +${STATS_FILE} ${CURDIR}/example_stats.csv +${FAILURE_FILE} ${CURDIR}/example_failures.csv +${HISTORY_FILE} ${CURDIR}/example_stats_history.csv +``` + + now we can run the robot tests from `packagenv/` folder with command: + + ``` +robot --listener oxygen.listener --pythonpath . test.robot +``` +and the tests should run normally. Now we have verified that the packaging has been done correctly. # Teardown From 3282226b08c83810fbfb4d4c6d812797cc29bdd3 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 7 Jul 2020 13:06:35 +0300 Subject: [PATCH 14/36] change variables when robot tests are run from different env --- DEVGUIDE.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 37c30a0..65db55d 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -634,8 +634,9 @@ locust.locusthandler: ``` - Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder , make the change above to the variables in `test.robot`: - ``` + Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder , make the following changes to the variables in `test.robot`: + +``` *** Variables *** ${STATS_FILE} ${CURDIR}/example_stats.csv ${FAILURE_FILE} ${CURDIR}/example_failures.csv @@ -644,7 +645,7 @@ ${HISTORY_FILE} ${CURDIR}/example_stats_history.csv now we can run the robot tests from `packagenv/` folder with command: - ``` +``` robot --listener oxygen.listener --pythonpath . test.robot ``` From 1c8adabb445a7438cdbfe01f95c57260d1fa208a Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 7 Jul 2020 13:18:30 +0300 Subject: [PATCH 15/36] add 'what is our goal' section --- DEVGUIDE.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 65db55d..c2beb3a 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -9,6 +9,35 @@ This is a developer guide for Oxygen. We will write a handler for [https://locus Python 3 +## What is our goal? + +The performance tests are defined in python files which look like following: + +``` +from locust import HttpUser, task, between + +class QuickstartUser(HttpUser): + wait_time = between(5000, 15000) + + @task + def index_page(self): + self.client.get("/") + +``` + +And the output of the tests are .csv files which look like following: + +``` +"Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" +"GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 +"POST","/",5,5,300,323,288,402,157,0.13,0.13,300,330,330,400,400,400,400,400,400,400,400,400 +"GET","/item",24,0,80,79,67,100,2175,0.63,0.00,81,85,86,86,89,92,100,100,100,100,100,100 +"None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 +``` + +Our goal is to write an handler, which is able to execute the locust tests inside robotframework and provide the test results in user-friendly format in the robot framework log files. + + ## Start developing Let's create a virtual environment and install oxygen. @@ -634,8 +663,8 @@ locust.locusthandler: ``` - Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder , make the following changes to the variables in `test.robot`: - + Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder, make the following changes to the variables in `test.robot`: + ``` *** Variables *** ${STATS_FILE} ${CURDIR}/example_stats.csv From fe52b05912c3829bdc6d9a50571f859ac8b5b9e1 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 7 Jul 2020 13:24:29 +0300 Subject: [PATCH 16/36] add comment to LocustHandlerException --- DEVGUIDE.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index c2beb3a..7c1f633 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -35,7 +35,7 @@ And the output of the tests are .csv files which look like following: "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -Our goal is to write an handler, which is able to execute the locust tests inside robotframework and provide the test results in user-friendly format in the robot framework log files. +Our goal is to write an handler, which is able to execute the locust tests inside robot framework and provide the test results in user-friendly format in the robot framework log files. ## Start developing @@ -80,15 +80,10 @@ class LocustHandler(BaseHandler): def run_locust(self, result_file, command, check_return_code=False, **env): - '''Run Locust performance testing tool with command - ``command``. - - See documentation for other arguments in \`Run Gatling\`. - ''' try: output = run_command_line(command, check_return_code, **env) except SubprocessException as e: - raise LocustHandlerException(e) + raise LocustHandlerException(e) ## It is best practice to raise LocustHandlerException so we know that LocustHandler caused the error. logger.info(output) logger.info('Result file: {}'.format(result_file)) return result_file @@ -455,15 +450,10 @@ from oxygen.utils import run_command_line, validate_path class LocustHandler(BaseHandler): def run_locust(self, result_file, command, check_return_code=False, failure_percentage=None, **env): - '''Run Locust performance testing tool with command - ``command``. - - See documentation for other arguments in \`Run Gatling\`. - ''' try: output = run_command_line(command, check_return_code, **env) except SubprocessException as e: - raise LocustHandlerException(e) + raise LocustHandlerException(e) ## It is best practice to raise LocustHandlerException so we know that LocustHandler caused the error. logger.info(output) logger.info('Result file: {}'.format(result_file)) dictionary = dict() From 3d41d32646893e3225c563ea979cea013e2c0ade Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Tue, 7 Jul 2020 16:26:37 +0300 Subject: [PATCH 17/36] add 'Improving the test result report' section --- DEVGUIDE.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 7c1f633..9d36455 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -671,6 +671,67 @@ robot --listener oxygen.listener --pythonpath . test.robot and the tests should run normally. Now we have verified that the packaging has been done correctly. +## Improving the test result report + +Our locusthandler works fine, but we could make the test results more clear. Let's change the `transform_tests` method of `locusthandler.py` to make more clear test suite and keyword names, and show the performance test results as keyword messages: + +``` + def _transform_tests(self, file, treshold_failure_percentage): + with open(file, newline='') as csvfile: + reader = csv.DictReader(csvfile) + test_cases = [] + for row in reader: + messages = [] + for element in row: + messages.append('{}: {}'.format(element, row[element])) + failure_count = row['Failure Count'] + request_count = row['Request Count'] + count_table = '| =Failure Count= | =Request Count=|\n| {} | {} |'.format(failure_count, request_count) + failure_percentage = 100 * int(failure_count) / int(request_count) + success = failure_percentage <= treshold_failure_percentage + keyword_name = '{} requests with {} failures '.format(request_count, failure_count) + keyword = { + 'name': keyword_name, + 'pass': success, + 'tags': [], + 'messages': messages, + 'teardown': [], + 'keywords': [], + } + type_of_request = row['Type'] + path = row['Name'] + test_case_name = 'Testing {} requests to path {}'.format(type_of_request, path) + if path == 'Aggregated': + test_case_name = 'Aggragated results of all Locust test cases:' + test_case = { + 'name': test_case_name, + 'tags': [], + 'setup': [], + 'teardown': [], + 'keywords': [keyword] + } + test_cases.append(test_case) + test_suite = { + 'name': 'Locust test case, failure percentage {}'.format(treshold_failure_percentage), + 'tags': self._tags, + 'setup': [], + 'teardown': [], + 'suites': [], + 'tests': test_cases, + } + return test_suite +``` + +now run the robot tests again from `locustenv/` folder with + +``` +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +``` + +and see the new test format in the generated`log.html` file. + + + # Teardown If you wish to delete your virtual environment do following: From 70d3c27082ad18070c8c5a3b70f90d6d41c7a5bf Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 11:12:51 +0300 Subject: [PATCH 18/36] rename locust folder to locusthandler and make small fixes --- DEVGUIDE.md | 85 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 9d36455..b995fde 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -56,14 +56,14 @@ $ pip install robotframework-oxygen Let's start developing by creating a working folder ```` cd locustenv -mkdir locust -cd locust +mkdir locusthandler +cd locusthandler ```` ### Writing LocustHandler and unit tests -Let's create `__init__.py` to our `locustenv/locust` folder. Next we can write `locusthandler.py` with following content: +Let's create `__init__.py` to our `locustenv/locusthandler` folder. Next we can create `locusthandler.py` to `locustenv/locusthandler` folder with following content: ``` import json @@ -129,9 +129,9 @@ class LocustHandlerException(Exception): pass ``` - Method `run_locust` can be used from robot tests, and it executes the command which runs the locust tests. It returns a path to the locust test results, which is processed by method `parse_results`, which calls `_transform_tests` function which purpose is to transfer the locust result file into a format which can be seen in the Robot Framework result files. + Method `run_locust` can be used from robot tests, and it executes the command which runs the locust tests. It returns a path to the locust test results, which is processed by method `parse_results`, which calls `_transform_tests` function which purpose is to transfer the locust result file into a format which can be seen in the Robot Framework log files. - Let's create a `tests` folder in `locustenv/locust` . Then we write test file `test_locust.py` with following content: + Let's create a `locustenv/locusthandler/tests` folder. Then we write there test file `test_locust.py` with following content: ``` from unittest import TestCase @@ -158,7 +158,7 @@ class TestLocust(TestCase): ``` -next we create `resources` folder in `locustenv/locust` and add there test data file `requests.csv` which has the following: +next we create `locustenv/locusthandler/resources` folder and add there test data file `requests.csv` which has the following: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" "GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 @@ -167,7 +167,7 @@ next we create `resources` folder in `locustenv/locust` and add there test data "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -Now we can run unit tests from the `locustenv/locust` folder with command +Now we can run unit tests from the `locustenv/locusthandler` folder with command ```` python -m unittest tests/test_locust.py @@ -182,13 +182,13 @@ Let's open the python interpreter from the `locustenv` directory and check that ``` python -import locust.locusthandler +import locusthandler.locusthandler ``` -running this should not produce any errors, and we can import file `locusthandler.py` from `/locust` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: +running this should not produce any errors, and we can import file `locusthandler.py` from `/locusthandler` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: ``` -locust.locusthandler: +locusthandler.locusthandler: handler: LocustHandler keyword: run_locust tags: oxygen-locusthandler @@ -219,7 +219,7 @@ First we install locust to our locustenv virtualenv: pip install locust ``` -Then we add `locustfile.py` file to `locustenv/locust` folder which contains the commands for the performance test: +Then we add `locustfile.py` file to `locustenv/locusthandler` folder which contains the commands for the performance test: ``` from locust import HttpUser, task, between @@ -235,7 +235,7 @@ class QuickstartUser(HttpUser): -Let's write `locustenv/locust/test.robot` file which contains test case that runs locust from command line: +Let's write `test.robot` file to `locustenv/locusthandler` folder which contains test case that runs locust from command line: ``` *** Settings *** @@ -276,7 +276,7 @@ My Locust test We can run the test from `locustenv` folder by using command ``` -robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locusthandler/locustfile.py locusthandler/test.robot ``` The test should execute for about 60 seconds. After this you can see the statistics of the performance tests in `log.html` and `report.html`. @@ -304,7 +304,7 @@ In our first solution the Locust test case will fail if even one request fails d Let's define the value of `failure_percentage` in `/lib/python3.7/site-packages/oxygen/config.yml`: ``` -locust.locusthandler: +locusthandler.locusthandler: handler: LocustHandler keyword: run_locust tags: oxygen-locusthandler @@ -312,7 +312,7 @@ locust.locusthandler: ``` -Let's implement function, which returns the failure_percentage to `locustenv/locust/locusthandler.py`: +Let's implement function, which returns the failure_percentage to `locustenv/locusthandler/locusthandler.py`: ``` def _get_treshold_failure_percentage(self): @@ -340,6 +340,7 @@ and let's use it in `_transform_tests` function: for row in reader: failure_count = row['Failure Count'] request_count = row['Request Count'] + failure_percentage = 100 * int(failure_count) / int(request_count) treshold_failure_percentage = self._get_treshold_failure_percentage() success = failure_percentage <= treshold_failure_percentage keyword = { @@ -369,7 +370,7 @@ and let's use it in `_transform_tests` function: return test_suite ``` -Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locust/resources/requests.csv`: +Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locusthandler/resources/requests.csv`: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" @@ -379,7 +380,7 @@ Let's update the tests to match the current functionality. Let's start by defini "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -now we can update the unit tests in `locust/tests/test_locust.py`: +now we can update the unit tests in `locusthandler/tests/test_locust.py`: ``` from unittest import TestCase @@ -418,7 +419,7 @@ class TestLocust(TestCase): self.assertEqual(failure_percentage, 100) ``` -Now the unit tests should pass, run the tests from `locustenv/locust` folder with command: +Now the unit tests should pass, run the tests from `locustenv/locusthandler` folder with command: ```` python -m unittest tests/test_locust.py @@ -427,14 +428,14 @@ python -m unittest tests/test_locust.py And we can also try out the robot tests using the new .yaml configuration. Run the tests from `locustenv` folder by using command ``` -robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locusthandler/locustfile.py locusthandler/test.robot ``` ## Adding failure percentage as an robot test parameter -Our current solution works quite nicely, but let's imagine a situation where we run the performance tests on different parts of the software, where we wan't to determine different values for the amount of `failure_percentage`. +Our current solution works quite nicely, but let's imagine a situation where we run the performance tests on different parts of the software, where we wan't to determine different values for `failure_percentage`. -Let's change the functionality of `locust/locusthandler.py`: +Let's change the functionality of `locusthandler/locusthandler.py`: ``` import json @@ -521,7 +522,7 @@ class LocustHandlerException(Exception): pass ``` -Notice that we return an dictionary object instead of result file in the `run_locust` method. This way we can use the `failure_percentage` value if it is defined. If it's not defined we will use the value what is defined in `/lib/python3.7/site-packages/oxygen/config.yml`. Now we can rewrite the robot tests in `locust/test.robot`, one assigns the value from the parameter and the second test doesn't: +Notice that we return an dictionary object instead of result file in the `run_locust` method. This way we can use the `failure_percentage` value if it is defined. If it's not defined we will use the value what is defined in `/lib/python3.7/site-packages/oxygen/config.yml`. Now we can rewrite the robot tests in `locusthandler/test.robot`, one assigns the value from the parameter and the second test doesn't: ``` *** Test Cases *** @@ -548,16 +549,16 @@ In this case the `Critical test` could be a performance test for a system where Run the tests in `locustenv/` folder with ``` -robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locusthandler/locustfile.py locusthandler/test.robot ``` -However now when you run the unit tests from `locust/` folder they fail: +However now when you run the unit tests from `locusthandler/` folder they fail: ``` python -m unittest tests/test_locust.py ``` -because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup and write a method `dictionary_with_result_file`: +because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup in `tests/test_locust.py` and write a method `dictionary_with_result_file`: ``` def dictionary_with_result_file(self): @@ -639,13 +640,13 @@ you should now have a version of locusthandler in your `packagenv` environment. ``` python -import locust.locusthandler +import locusthandler.locusthandler ``` which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: ``` -locust.locusthandler: +locusthandler.locusthandler: handler: LocustHandler keyword: run_locust tags: oxygen-locusthandler @@ -725,12 +726,38 @@ Our locusthandler works fine, but we could make the test results more clear. Let now run the robot tests again from `locustenv/` folder with ``` -robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust/locustfile.py locust/test.robot +robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locusthandler/locustfile.py locusthandler/test.robot ``` -and see the new test format in the generated`log.html` file. +and see the new test format in the generated `log.html` file. + +Now when we see the unit tests from `locustenv/locusthandler` folder: + +``` +python -m unittest tests/test_locust.py +``` + We notice that two fail because we changed the name of test suite. Let's update the assert statements of the failing tests: +``` + def test_parse_results_takes_failure_percentage_from_parameter_prior_to_config(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '70'} + self.handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() + dictionary['failure_percentage'] = 75 + test_suite = self.handler.parse_results(dictionary) + self.assertEqual(test_suite['name'], 'Locust test case, failure percentage 75') + + def test_parse_results_takes_failure_percentage_correctly_from_config(self): + config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '70'} + self.handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() + dictionary['failure_percentage'] = None + test_suite = self.handler.parse_results(dictionary) + self.assertEqual(test_suite['name'], 'Locust test case, failure percentage 70') +``` + +And we are done! # Teardown From ad2c83e35c7429992490b272568cf38047de7954 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 12:07:22 +0300 Subject: [PATCH 19/36] make python intepreter parts more clear --- DEVGUIDE.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index b995fde..e9e7ce8 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -181,8 +181,10 @@ and all 3 tests should pass. Let's open the python interpreter from the `locustenv` directory and check that we can import the locusthandler: ``` -python -import locusthandler.locusthandler +$ python +Python 3.7.7 +Type "help", "copyright", "credits" or "license" for more information. +>>> import locusthandler.locusthandler ``` running this should not produce any errors, and we can import file `locusthandler.py` from `/locusthandler` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: @@ -639,8 +641,10 @@ pip install locustenv/dist/NAME-OF-YOUR-PACKAGE.whl you should now have a version of locusthandler in your `packagenv` environment. Let's verify this by opening python intepreter: ``` -python -import locusthandler.locusthandler +$ python +Python 3.7.7 +Type "help", "copyright", "credits" or "license" for more information. +>>> import locusthandler.locusthandler ``` which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: From ef9e6a754212e56167d9d087eaf302cd6740d0fa Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 12:10:39 +0300 Subject: [PATCH 20/36] remove venv and install command when running the demoapp --- DEVGUIDE.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index e9e7ce8..775b493 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -206,11 +206,7 @@ Open up another terminal and run following commands: ``` git clone https://github.com/robotframework/WebDemo.git cd WebDemo -python3 -m venv myenv -source myenv/bin/activate - -pip install -r requirements.txt -python demoapp/server.py +python3 demoapp/server.py ``` ### Running Locust with LocustHandler in Robot test From 2b0e5fe3dfb2cc22a15816a69a07a720dbff8931 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 12:18:48 +0300 Subject: [PATCH 21/36] rename robot test cases --- DEVGUIDE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 775b493..e282238 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -262,8 +262,8 @@ Remove csv files *** Test Cases *** -My Locust test - [Tags] LOCUST_ROBOT_TAG +Performance test should pass + [Tags] performance-tests Remove csv files run_locust ... ${STATS_FILE} @@ -525,8 +525,8 @@ Notice that we return an dictionary object instead of result file in the `run_lo ``` *** Test Cases *** -Critical test - [Tags] LOCUST_ROBOT_TAG +Critical performance test + [Tags] performance-tests Remove csv files run_locust ... ${STATS_FILE} @@ -534,8 +534,8 @@ Critical test ... failure_percentage=${1} -Normal test - [Tags] LOCUST_ROBOT_TAG +Normal performance test + [Tags] performance-tests Remove csv files run_locust ... ${STATS_FILE} From 89cf4b1eaf9f8d86f1a62bdd351e69f7f26a1a21 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 12:25:23 +0300 Subject: [PATCH 22/36] call keyword with Run Locust instead of run_locust in robot test cases --- DEVGUIDE.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index e282238..7393468 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -265,7 +265,7 @@ Remove csv files Performance test should pass [Tags] performance-tests Remove csv files - run_locust + Run Locust ... ${STATS_FILE} ... ${LOCUSTCOMMAND} ``` @@ -285,10 +285,10 @@ If the test case fails, you can set variable `check_return_code` to "True" in or ``` *** Test Cases *** -My Locust test - [Tags] LOCUST_ROBOT_TAG +Performance test should pass + [Tags] performance-tests Remove csv files - run_locust + Run Locust ... ${STATS_FILE} ... ${LOCUSTCOMMAND} ... check_return_code=${True} @@ -528,7 +528,7 @@ Notice that we return an dictionary object instead of result file in the `run_lo Critical performance test [Tags] performance-tests Remove csv files - run_locust + Run Locust ... ${STATS_FILE} ... ${LOCUSTCOMMAND} ... failure_percentage=${1} @@ -537,7 +537,7 @@ Critical performance test Normal performance test [Tags] performance-tests Remove csv files - run_locust + Run Locust ... ${STATS_FILE} ... ${LOCUSTCOMMAND} ``` From bb24b042fef6338ec0f9f89df6c8361be0bbb6d0 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 15:00:31 +0300 Subject: [PATCH 23/36] return empty array instead of self._tags --- DEVGUIDE.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 7393468..2b54abd 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -117,7 +117,7 @@ class LocustHandler(BaseHandler): test_cases.append(test_case) test_suite = { 'name': 'Locust Scenario', - 'tags': self._tags, + 'tags': [], 'setup': [], 'teardown': [], 'suites': [], @@ -359,7 +359,7 @@ and let's use it in `_transform_tests` function: test_cases.append(test_case) test_suite = { 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), - 'tags': self._tags, + 'tags': [], 'setup': [], 'teardown': [], 'suites': [], @@ -495,7 +495,7 @@ class LocustHandler(BaseHandler): test_cases.append(test_case) test_suite = { 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), - 'tags': self._tags, + 'tags': [], 'setup': [], 'teardown': [], 'suites': [], @@ -714,7 +714,7 @@ Our locusthandler works fine, but we could make the test results more clear. Let test_cases.append(test_case) test_suite = { 'name': 'Locust test case, failure percentage {}'.format(treshold_failure_percentage), - 'tags': self._tags, + 'tags': [], 'setup': [], 'teardown': [], 'suites': [], From d7c6ad0b99cb6084f4a79e1e63fc34819015c1ae Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 15:40:20 +0300 Subject: [PATCH 24/36] remove unneccessary empty arrays from locusthandler.py --- DEVGUIDE.md | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 2b54abd..a0853b3 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -102,25 +102,14 @@ class LocustHandler(BaseHandler): keyword = { 'name': " | ".join(row), 'pass': success, - 'tags': [], - 'messages': [], - 'teardown': [], - 'keywords': [], } test_case = { 'name': 'Locust test case', - 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [keyword] } test_cases.append(test_case) test_suite = { 'name': 'Locust Scenario', - 'tags': [], - 'setup': [], - 'teardown': [], - 'suites': [], 'tests': test_cases, } return test_suite @@ -344,25 +333,14 @@ and let's use it in `_transform_tests` function: keyword = { 'name': " | ".join(row), 'pass': success, - 'tags': [], - 'messages': [], - 'teardown': [], - 'keywords': [], } test_case = { 'name': 'Locust test case', - 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [keyword] } test_cases.append(test_case) test_suite = { 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), - 'tags': [], - 'setup': [], - 'teardown': [], - 'suites': [], 'tests': test_cases, } return test_suite @@ -480,25 +458,14 @@ class LocustHandler(BaseHandler): keyword = { 'name': " | ".join(row), 'pass': success, - 'tags': [], - 'messages': [], - 'teardown': [], - 'keywords': [], } test_case = { 'name': 'Locust test case', - 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [keyword] } test_cases.append(test_case) test_suite = { 'name': 'Locust test suite, failure percentage {}'.format(treshold_failure_percentage), - 'tags': [], - 'setup': [], - 'teardown': [], - 'suites': [], 'tests': test_cases, } return test_suite @@ -694,10 +661,7 @@ Our locusthandler works fine, but we could make the test results more clear. Let keyword = { 'name': keyword_name, 'pass': success, - 'tags': [], 'messages': messages, - 'teardown': [], - 'keywords': [], } type_of_request = row['Type'] path = row['Name'] @@ -706,18 +670,11 @@ Our locusthandler works fine, but we could make the test results more clear. Let test_case_name = 'Aggragated results of all Locust test cases:' test_case = { 'name': test_case_name, - 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [keyword] } test_cases.append(test_case) test_suite = { 'name': 'Locust test case, failure percentage {}'.format(treshold_failure_percentage), - 'tags': [], - 'setup': [], - 'teardown': [], - 'suites': [], 'tests': test_cases, } return test_suite From f54d44b176cdc269a0126e0540765856fccd0a78 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 15:54:40 +0300 Subject: [PATCH 25/36] change python intepreter parts --- DEVGUIDE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index a0853b3..3863c96 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -167,13 +167,14 @@ and all 3 tests should pass. ### Configuring LocustHandler to Oxygen -Let's open the python interpreter from the `locustenv` directory and check that we can import the locusthandler: +Let's open the python interpreter by running `python` from the `locustenv` directory and check that we can import the locusthandler: ``` -$ python +(locustenv) $ python Python 3.7.7 Type "help", "copyright", "credits" or "license" for more information. >>> import locusthandler.locusthandler +>>> ``` running this should not produce any errors, and we can import file `locusthandler.py` from `/locusthandler` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: @@ -608,6 +609,7 @@ $ python Python 3.7.7 Type "help", "copyright", "credits" or "license" for more information. >>> import locusthandler.locusthandler +>>> ``` which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: From d9c417aef704473b9ebda437de3b78f15863e947 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 16:03:46 +0300 Subject: [PATCH 26/36] explain the purpose of new data set --- DEVGUIDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 3863c96..06ae0a3 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -347,7 +347,7 @@ and let's use it in `_transform_tests` function: return test_suite ``` -Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locusthandler/resources/requests.csv`: +Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locusthandler/resources/requests.csv`, which has 30 requests and 3 failed requests in the third data row: ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" @@ -357,7 +357,7 @@ Let's update the tests to match the current functionality. Let's start by defini "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -now we can update the unit tests in `locusthandler/tests/test_locust.py`: +now we can update the unit tests in `locusthandler/tests/test_locust.py` to test that the pass value is calculated correctly depending on the value of `failure_precentage`: ``` from unittest import TestCase From e6e2f4cc85701a7ee93ae4f9c30c920630c66bf2 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 16:08:09 +0300 Subject: [PATCH 27/36] unit tests refer to default value instead of magic number --- DEVGUIDE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 06ae0a3..b6f85aa 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -376,13 +376,13 @@ class TestLocust(TestCase): def test_suite_has_four_cases(self): self.assertEqual(len(self.test_suite['tests']),4) - def test_pass_is_true_when_failure_request_percentage_is_zero(self): + def test_pass_is_true_when_failure_request_percentage_is_below_default_value(self): self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) - def test_pass_is_true_when_failure_request_percentage_is_ten(self): + def test_pass_is_true_when_failure_request_percentage_is_default_value(self): self.assertEqual(self.test_suite['tests'][2]['keywords'][0]['pass'], True) - def test_pass_is_false_when_failure_request_percentage_is_above_ten(self): + def test_pass_is_false_when_failure_request_percentage_is_above_default_value(self): self.assertEqual(self.test_suite['tests'][1]['keywords'][0]['pass'], False) def test_failure_percentage_is_ten_by_default(self): From 7deba55c675de9d284cfe33cd0f3cce94d90079c Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 16:17:35 +0300 Subject: [PATCH 28/36] remove installation of wheel and setuptools --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index b6f85aa..79a7c59 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -579,7 +579,7 @@ All 8 tests should pass. Now we have completed an LocustHandler with unit tests ## How to package your project -Let's package our project in the same virtual environment . Add necessary files defined in this [tutorial](https://packaging.python.org/) to `locustenv` folder and install `setuptools` and `wheel` to your virtualenv with `pip install`. Add following line yo your `setup.py` file in the `setuptools.setup() object`: +Let's package our project in the same virtual environment . Add necessary files defined in this [tutorial](https://packaging.python.org/) to `locustenv` folder and add following line yo your `setup.py` file in the `setuptools.setup() object`: ``` install_requires=[ From 48b184e99be935e17803f8b6c6bc625d4f6c4b6a Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Wed, 8 Jul 2020 16:21:39 +0300 Subject: [PATCH 29/36] explain copying of files to packagenv environment --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 79a7c59..4e0fb21 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -623,7 +623,7 @@ locusthandler.locusthandler: ``` - Next let's run the robot test case to make sure that it works. Copy `test.robot` and `locustfile.py` files to `packagenv/` folder, make the following changes to the variables in `test.robot`: + Next let's run the robot test case to make sure that it works. Next let's copy `test.robot` and `locustfile.py` files to `packagenv/` folder so that we can run them easily from our new environment. Make the following changes to the variables in `test.robot`: ``` *** Variables *** From 2061d2b22ee4c241cb1da850f4373afa542e5eda Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 10:20:31 +0300 Subject: [PATCH 30/36] remove failure_precentage_is_ten_by_default test, install wheel when packaging, fix typos --- DEVGUIDE.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 4e0fb21..312dd60 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -357,7 +357,7 @@ Let's update the tests to match the current functionality. Let's start by defini "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -now we can update the unit tests in `locusthandler/tests/test_locust.py` to test that the pass value is calculated correctly depending on the value of `failure_precentage`: +now we can update the unit tests in `locusthandler/tests/test_locust.py` to test that the pass value is calculated correctly depending on the value of `failure_percentage`: ``` from unittest import TestCase @@ -385,10 +385,6 @@ class TestLocust(TestCase): def test_pass_is_false_when_failure_request_percentage_is_above_default_value(self): self.assertEqual(self.test_suite['tests'][1]['keywords'][0]['pass'], False) - def test_failure_percentage_is_ten_by_default(self): - failure_percentage = self.handler._get_treshold_failure_percentage() - self.assertEqual(failure_percentage, 10) - def test_failure_percentage_max_amount_is_one_hundred(self): config = config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '101'} self.handler = LocustHandler(config) @@ -541,13 +537,9 @@ because we changed the functionality to use dictionary instead of result file pa self.test_suite = self.handler.parse_results(dictionary) ``` -and run tests again. Still two test cases fail. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the failing test cases: +and run tests again. Still one test case fails. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the failing test case: ``` - def test_failure_percentage_is_ten_by_default(self): - failure_percentage = self.handler._get_treshold_failure_percentage(None) - self.assertEqual(failure_percentage, 10) - def test_failure_percentage_max_amount_is_one_hundred(self): failure_percentage = self.handler._get_treshold_failure_percentage(101) self.assertEqual(failure_percentage, 100) @@ -573,7 +565,7 @@ Now all tests should pass. Let's add two more test cases to see that the `failur self.assertEqual(test_suite['name'], 'Locust test suite, failure percentage 70') ``` -All 8 tests should pass. Now we have completed an LocustHandler with unit tests which test the most important functionalities. +All tests should pass. Now we have completed an LocustHandler with unit tests which test the most important functionalities. @@ -591,6 +583,7 @@ Let's package our project in the same virtual environment . Add necessary files so that oxygen including it's dependencies and locust will be installed when your handler is installed. Next you can run following command from `locustenv` folder: ``` +pip install wheel python setup.py sdist bdist_wheel ``` @@ -656,7 +649,6 @@ Our locusthandler works fine, but we could make the test results more clear. Let messages.append('{}: {}'.format(element, row[element])) failure_count = row['Failure Count'] request_count = row['Request Count'] - count_table = '| =Failure Count= | =Request Count=|\n| {} | {} |'.format(failure_count, request_count) failure_percentage = 100 * int(failure_count) / int(request_count) success = failure_percentage <= treshold_failure_percentage keyword_name = '{} requests with {} failures '.format(request_count, failure_count) From 6aeef8b617967ac854e8929cb6c5a1b467efec6d Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 12:48:21 +0300 Subject: [PATCH 31/36] remove sdist when packaging --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 312dd60..e43c57f 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -584,7 +584,7 @@ so that oxygen including it's dependencies and locust will be installed when you ``` pip install wheel -python setup.py sdist bdist_wheel +python setup.py bdist_wheel ``` which will create you a `locustenv/dist` folder. Next we will ensure that the installation works by creating another virtualenv. Open up another terminal, go backwards with `cd ..` same path where `locustenv` is and run following commands: From 6e72553439250638748fa310377828c6af533ad4 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 13:13:10 +0300 Subject: [PATCH 32/36] remind to check out config.yml if robot test case fails --- DEVGUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index e43c57f..06e65c3 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -270,7 +270,7 @@ robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locust The test should execute for about 60 seconds. After this you can see the statistics of the performance tests in `log.html` and `report.html`. -If the test case fails, you can set variable `check_return_code` to "True" in order to get more specific logging: +If the test case fails, check first that Oxygen's `config.yml` is correctly configured from the previous section. You can set variable `check_return_code` to "True" in order to get more specific logging: ``` *** Test Cases *** From a496a3b75a7ae0cad565581d6db70a3901bf2618 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 13:15:08 +0300 Subject: [PATCH 33/36] add instructions to check out is the yaml change correct --- DEVGUIDE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 06e65c3..cc46912 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -186,6 +186,14 @@ locusthandler.locusthandler: tags: oxygen-locusthandler ``` +Test your edit by running: + +``` +$ python -m oxygen --version +``` + +You shouldn't get any errors. If you do, check that your edits are valid [YAML](https://yaml.org/) syntax. + ### Install demoapp to run tests against [Next we we install and run demo-app that we run the locust tests against.](https://github.com/robotframework/WebDemo) From f7636b1268340da4a71f1e6edee78d6b053e3c9b Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 13:24:55 +0300 Subject: [PATCH 34/36] add syntax highlights for RobotFramework parts --- DEVGUIDE.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index cc46912..d149888 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -233,7 +233,8 @@ class QuickstartUser(HttpUser): Let's write `test.robot` file to `locustenv/locusthandler` folder which contains test case that runs locust from command line: -``` +```RobotFramework + *** Settings *** Library oxygen.OxygenLibrary Library OperatingSystem @@ -280,7 +281,7 @@ The test should execute for about 60 seconds. After this you can see the statist If the test case fails, check first that Oxygen's `config.yml` is correctly configured from the previous section. You can set variable `check_return_code` to "True" in order to get more specific logging: -``` +```RobotFramework *** Test Cases *** Performance test should pass @@ -494,7 +495,7 @@ class LocustHandlerException(Exception): Notice that we return an dictionary object instead of result file in the `run_locust` method. This way we can use the `failure_percentage` value if it is defined. If it's not defined we will use the value what is defined in `/lib/python3.7/site-packages/oxygen/config.yml`. Now we can rewrite the robot tests in `locusthandler/test.robot`, one assigns the value from the parameter and the second test doesn't: -``` +```RobotFramework *** Test Cases *** Critical performance test @@ -626,7 +627,7 @@ locusthandler.locusthandler: Next let's run the robot test case to make sure that it works. Next let's copy `test.robot` and `locustfile.py` files to `packagenv/` folder so that we can run them easily from our new environment. Make the following changes to the variables in `test.robot`: -``` +```RobotFramework *** Variables *** ${STATS_FILE} ${CURDIR}/example_stats.csv ${FAILURE_FILE} ${CURDIR}/example_failures.csv From e9b3b25962c1427ad2a13bd081aa6275df034268 Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 14:00:39 +0300 Subject: [PATCH 35/36] change default failure_precentage from ten to zero --- DEVGUIDE.md | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index d149888..626ee6e 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -226,7 +226,6 @@ class QuickstartUser(HttpUser): @task def index_page(self): self.client.get("/") - ``` @@ -282,6 +281,7 @@ The test should execute for about 60 seconds. After this you can see the statist If the test case fails, check first that Oxygen's `config.yml` is correctly configured from the previous section. You can set variable `check_return_code` to "True" in order to get more specific logging: ```RobotFramework + *** Test Cases *** Performance test should pass @@ -316,8 +316,8 @@ Let's implement function, which returns the failure_percentage to `locustenv/loc failure_percentage = self._config.get('failure_percentage', None) if failure_percentage is None: - print('No failure percentage configured, defaulting to 10') - return 10 + print('No failure percentage configured, defaulting to 0') + return 0 failure_percentage = int(failure_percentage) @@ -356,17 +356,9 @@ and let's use it in `_transform_tests` function: return test_suite ``` -Let's update the tests to match the current functionality. Let's start by defining new data set in `locustenv/locusthandler/resources/requests.csv`, which has 30 requests and 3 failed requests in the third data row: -``` -"Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" -"GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 -"POST","/",5,5,300,323,288,402,157,0.13,0.13,300,330,330,400,400,400,400,400,400,400,400,400 -"GET","/item",30,3,80,79,67,100,2175,0.63,0.00,81,85,86,86,89,92,100,100,100,100,100,100 -"None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 -``` -now we can update the unit tests in `locusthandler/tests/test_locust.py` to test that the pass value is calculated correctly depending on the value of `failure_percentage`: +Next we can update the unit tests in `locusthandler/tests/test_locust.py` to test that the pass value is calculated correctly depending on the value of `failure_percentage`: ``` from unittest import TestCase @@ -386,6 +378,10 @@ class TestLocust(TestCase): self.assertEqual(len(self.test_suite['tests']),4) def test_pass_is_true_when_failure_request_percentage_is_below_default_value(self): + config = config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '10'} + handler = LocustHandler(config) + path = Path.cwd() / 'resources/requests.csv' + test_suite = handler.parse_results(path) self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) def test_pass_is_true_when_failure_request_percentage_is_default_value(self): @@ -396,8 +392,8 @@ class TestLocust(TestCase): def test_failure_percentage_max_amount_is_one_hundred(self): config = config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST', 'failure_percentage': '101'} - self.handler = LocustHandler(config) - failure_percentage = self.handler._get_treshold_failure_percentage() + handler = LocustHandler(config) + failure_percentage = handler._get_treshold_failure_percentage() self.assertEqual(failure_percentage, 100) ``` @@ -478,8 +474,8 @@ class LocustHandler(BaseHandler): def _get_treshold_failure_percentage(self, failure_percentage): if failure_percentage is None: - print('No failure percentage configured, defaulting to 10') - return 10 + print('No failure percentage configured, defaulting to 0') + return 0 failure_percentage = int(failure_percentage) @@ -546,9 +542,17 @@ because we changed the functionality to use dictionary instead of result file pa self.test_suite = self.handler.parse_results(dictionary) ``` -and run tests again. Still one test case fails. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the failing test case: +and run tests again. Still two test cases fail. This is because the `_get_treshold_failure_percentage` has an argument now instead of reading the value from the config. Let's update the failing test cases: ``` + def test_pass_is_true_when_failure_request_percentage_is_below_default_value(self): + config = config = {'handler': 'LocustHandler', 'keyword': 'run_locust', 'tags': 'LOCUST'} + handler = LocustHandler(config) + dictionary = self.dictionary_with_result_file() + dictionary['failure_percentage'] = 10 + test_suite = handler.parse_results(dictionary) + self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) + def test_failure_percentage_max_amount_is_one_hundred(self): failure_percentage = self.handler._get_treshold_failure_percentage(101) self.assertEqual(failure_percentage, 100) From 24986ce9577bdc02c3204332e8d5ae348b95627a Mon Sep 17 00:00:00 2001 From: Aleksi Mustonen Date: Thu, 9 Jul 2020 14:11:45 +0300 Subject: [PATCH 36/36] fix typos --- DEVGUIDE.md | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 626ee6e..892a585 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -35,7 +35,7 @@ And the output of the tests are .csv files which look like following: "None","Aggregated",39,5,81,109,66,402,1916,1.03,0.13,81,86,87,89,300,330,400,400,400,400,400,400 ``` -Our goal is to write an handler, which is able to execute the locust tests inside robot framework and provide the test results in user-friendly format in the robot framework log files. +Our goal is to write an handler, which is able to execute the locust tests inside Robot Framework and provide the test results in user-friendly format in the Robot Framework log files. ## Start developing @@ -49,6 +49,7 @@ source locustenv/bin/activate ``` Install Oxygen by running the following: + ``` $ pip install robotframework-oxygen ``` @@ -147,7 +148,8 @@ class TestLocust(TestCase): ``` -next we create `locustenv/locusthandler/resources` folder and add there test data file `requests.csv` which has the following: +Next we create `locustenv/locusthandler/resources` folder and add there test data file `requests.csv` which has the following: + ``` "Type","Name","Request Count","Failure Count","Median Response Time","Average Response Time","Min Response Time","Max Response Time","Average Content Size","Requests/s","Failures/s","50%","66%","75%","80%","90%","95%","98%","99%","99.9%","99.99%","99.999%","100%" "GET","/",10,0,72,75,66,89,2175,0.26,0.00,73,75,86,87,89,89,89,89,89,89,89,89 @@ -177,7 +179,7 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -running this should not produce any errors, and we can import file `locusthandler.py` from `/locusthandler` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: +Running this should not produce any errors, and we can import file `locusthandler.py` from `/locusthandler` folder we created. [Read more about packaging python projects from here.](https://packaging.python.org/glossary/#term-import-package) Next we can exit the python intepreter (CTRL + D) and write following lines to the end of `lib/python3.7/site-packages/oxygen/config.yml`: ``` locusthandler.locusthandler: @@ -209,7 +211,7 @@ python3 demoapp/server.py ### Running Locust with LocustHandler in Robot test -First we install locust to our locustenv virtualenv: +First we install Locust to our locustenv virtualenv: ``` pip install locust @@ -511,7 +513,7 @@ Normal performance test ... ${LOCUSTCOMMAND} ``` -In this case the `Critical test` could be a performance test for a system where the consequences of failure is much larger: Thus we define the failure_percentage to 1%. In the `Normal test` we use the value that is defined in the `/lib/python3.7/site-packages/oxygen/config.yml`. +In this case the `Critical performance test` could be a performance test for a system where the consequences of failure is much larger: Thus we define the failure_percentage to 1%. In the `Normal performance test` we use the value that is defined in the `/lib/python3.7/site-packages/oxygen/config.yml`. Run the tests in `locustenv/` folder with @@ -525,7 +527,7 @@ However now when you run the unit tests from `locusthandler/` folder they fail: python -m unittest tests/test_locust.py ``` -because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup in `tests/test_locust.py` and write a method `dictionary_with_result_file`: +Because we changed the functionality to use dictionary instead of result file path. Let's update the test case setup in `tests/test_locust.py` and write a method `dictionary_with_result_file`: ``` def dictionary_with_result_file(self): @@ -552,7 +554,7 @@ and run tests again. Still two test cases fail. This is because the `_get_tresho dictionary['failure_percentage'] = 10 test_suite = handler.parse_results(dictionary) self.assertEqual(self.test_suite['tests'][0]['keywords'][0]['pass'], True) - + def test_failure_percentage_max_amount_is_one_hundred(self): failure_percentage = self.handler._get_treshold_failure_percentage(101) self.assertEqual(failure_percentage, 100) @@ -593,14 +595,14 @@ Let's package our project in the same virtual environment . Add necessary files ], ``` -so that oxygen including it's dependencies and locust will be installed when your handler is installed. Next you can run following command from `locustenv` folder: +So that Oxygen including it's dependencies and Locust will be installed when your handler is installed. Next you can run following command from `locustenv` folder: ``` pip install wheel python setup.py bdist_wheel ``` -which will create you a `locustenv/dist` folder. Next we will ensure that the installation works by creating another virtualenv. Open up another terminal, go backwards with `cd ..` same path where `locustenv` is and run following commands: +Which will create you a `locustenv/dist` folder. Next we will ensure that the installation works by creating another virtualenv. Open up another terminal, go backwards with `cd ..` same path where `locustenv` is and run following commands: ``` python3 -m venv packagenv @@ -608,7 +610,7 @@ source packagenv/bin/activate pip install locustenv/dist/NAME-OF-YOUR-PACKAGE.whl ``` -you should now have a version of locusthandler in your `packagenv` environment. Let's verify this by opening python intepreter: +You should now have a version of locusthandler in your `packagenv` environment. Let's verify this by opening python intepreter: ``` $ python @@ -618,7 +620,7 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: +Which should succeed. Exit intepreter with CTRL+ D. Next we can add following to the `packagenv/lib/python3.7/site-packages/oxygen/config.yml` file: ``` locusthandler.locusthandler: @@ -638,13 +640,13 @@ ${FAILURE_FILE} ${CURDIR}/example_failures.csv ${HISTORY_FILE} ${CURDIR}/example_stats_history.csv ``` - now we can run the robot tests from `packagenv/` folder with command: +Now we can run the robot tests from `packagenv/` folder with command: ``` robot --listener oxygen.listener --pythonpath . test.robot ``` -and the tests should run normally. Now we have verified that the packaging has been done correctly. +And the tests should run normally. Now we have verified that the packaging has been done correctly. ## Improving the test result report @@ -687,15 +689,15 @@ Our locusthandler works fine, but we could make the test results more clear. Let return test_suite ``` -now run the robot tests again from `locustenv/` folder with +Now run the robot tests again from `locustenv/` folder with ``` robot --listener oxygen.listener --pythonpath . --variable LOCUSTFILEPATH:locusthandler/locustfile.py locusthandler/test.robot ``` -and see the new test format in the generated `log.html` file. +And see the new test format in the generated `log.html` file. -Now when we see the unit tests from `locustenv/locusthandler` folder: +Let's run the unit tests from `locustenv/locusthandler` folder: ``` python -m unittest tests/test_locust.py @@ -732,4 +734,4 @@ deactivate rm -rf locustenv ``` -And shutdown the demo-app which was tested by locust with CTRL+D. You can also deactivate and delete `myenv` virtual environment if you wish. \ No newline at end of file +And shutdown the demo-app which was tested by locust with CTRL+D. You can also deactivate and delete `packagenv` virtual environment if you wish. \ No newline at end of file