RedisGraph benchmark automation (#1557)

* Added local and remote benchmark definition and automation

* [fix] Fixes per PR review. Added option to specify benchmark via BENCHMARH=<benchmark name>. Updated benchmark template

Co-authored-by: filipecosta90 <[email protected]>

Author: swilly22

.circleci/config.yml

@@ -22,17 +22,35 @@ commands:
             (mkdir -p deps; cd deps; git clone https://github.com/RedisLabsModules/readies.git)
             PIP=1 ./deps/readies/bin/getpy3
             python3 -m pip install -r ./deps/readies/paella/requirements.txt
+  
+  load-cached-deps:
+    steps:
+      # Load GraphBLAS from cache if possible.
+      - restore_cache:
+          keys:
+          - GraphBLAS4.0-{{checksum "./deps/GraphBLAS/README.md"}}
 
-jobs:
-  build:
-    docker:
-      - image: 'redisfab/rmbuilder:6.0.9-x64-bionic'
-        environment:
-            LANG: en_US.UTF-8  
-            LANGUAGE: en_US.UTF-8
-            LC_ALL: en_US.UTF-8     
+      # Load libcypher-parser from cache if possible.
+      - restore_cache:
+          keys:
+          - libcypher-parser-{{checksum "./deps/libcypher-parser/README.md"}}
+  
+  save-deps-cache:
+    steps:
+      # Save GraphBLAS to cache.
+      - save_cache:
+          paths:
+            - deps/GraphBLAS
+          key: GraphBLAS4.0-{{checksum "./deps/GraphBLAS/README.md"}}
+
+      # Save libcypher-parser to cache.
+      - save_cache:
+          paths:
+            - deps/libcypher-parser
+          key: libcypher-parser-{{checksum "./deps/libcypher-parser/README.md"}}
+
+  setup-prerequisits:
     steps:
-      - checkout
       - run:
           name: Fetch submodules
           command: |
@@ -59,49 +77,34 @@ jobs:
               locale-gen --purge en_US.UTF-8
               dpkg-reconfigure -f noninteractive locales
 
+jobs:
+  build:
+    docker:
+      - image: 'redisfab/rmbuilder:6.0.9-x64-bionic'
+        environment:
+            LANG: en_US.UTF-8  
+            LANGUAGE: en_US.UTF-8
+            LC_ALL: en_US.UTF-8  
+    steps:
+      - checkout
+      - setup-prerequisits
       - setup-automation
-
-      # Validate source complies with formatting
-      #- run:
-          # name: Formatting
-          # command: astyle --options=.astylerc -R "./*.c,*.h" --dry-run -Q
-
-      # Load GraphBLAS from cache if possible.
-      - restore_cache:
-          keys:
-          - GraphBLAS4.0-{{checksum "./deps/GraphBLAS/README.md"}}
-
-      # Load libcypher-parser from cache if possible.
-      - restore_cache:
-          keys:
-          - libcypher-parser-{{checksum "./deps/libcypher-parser/lib/src/parser.leg"}}
+      - load-cached-deps
 
       - run:
           name: Build
           command: make
 
-      # Save GraphBLAS to cache.
-      - save_cache:
-          paths:
-            - deps/GraphBLAS
-          key: GraphBLAS4.0-{{checksum "./deps/GraphBLAS/README.md"}}
+      - save-deps-cache
 
-      # Save libcypher-parser to cache.
-      - save_cache:
-          paths:
-            - deps/libcypher-parser
-          key: libcypher-parser-{{checksum "./deps/libcypher-parser/README.md"}}
-
-      - run:
-          name: Test
-          command: make test
       - early_return_for_forked_pull_requests
 
       - run:
           name: Persist Artifacts
           command: |
             strip src/$MODULE_ARTIFACT
-            mkdir -p /workspace/build && cp src/$MODULE_ARTIFACT /workspace
+            mkdir -p /workspace/build
+            cp src/$MODULE_ARTIFACT /workspace
             cp ramp.yml /workspace/
 
       - persist_to_workspace:
@@ -111,12 +114,45 @@ jobs:
             - ramp.yml
             - build
 
+  build_test:
+    docker:
+      - image: 'redisfab/rmbuilder:6.0.9-x64-bionic'
+        environment:
+            LANG: en_US.UTF-8  
+            LANGUAGE: en_US.UTF-8
+            LC_ALL: en_US.UTF-8
+    steps:
+      - checkout
+      - setup-prerequisits
+      - setup-automation
+      - attach_workspace:
+          at: /workspace
+      - load-cached-deps
+
+      - run:
+          name: Test
+          command: make test
+
+  build_memcheck:
+    docker:
+      - image: 'redisfab/rmbuilder:6.0.9-x64-bionic'
+        environment:
+            LANG: en_US.UTF-8  
+            LANGUAGE: en_US.UTF-8
+            LC_ALL: en_US.UTF-8
+    steps:
+      - checkout
+      - setup-prerequisits
+      - setup-automation
+      - load-cached-deps
+      - attach_workspace:
+          at: /workspace
+
       - run:
           name: Test for memory leaks
           command: |
               # Replace the default Redis server with one linked to libc malloc rather than jemalloc.
               python ./deps/readies/bin/getredis --force --valgrind
-              make clean
               make memcheck # Re-run the test suite, failing if definite memory leaks have been introduced.
           # Allow RediSearch global destructors.
           environment:
@@ -208,6 +244,41 @@ jobs:
           name: Run QA Automation
           command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
 
+  performance_ci_automation:
+    docker:
+      - image: 'redisfab/rmbuilder:6.0.9-x64-bionic'
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /workspace
+      - run:
+          name: Install remote benchmark tool dependencies
+          command: TF_EXE_FILE_NAME=/workspace/terraform ./tests/benchmarks/remote/install_deps.sh
+      - run:
+          name: Install remote benchmark python dependencies
+          command: python3 -m pip install -r ./tests/benchmarks/requirements.txt
+      - run:
+          name: Run CI benchmarks on aws
+          command: |
+              cd ./tests/benchmarks
+              export AWS_ACCESS_KEY_ID=$PERFORMANCE_EC2_ACCESS_KEY
+              export AWS_SECRET_ACCESS_KEY=$PERFORMANCE_EC2_SECRET_KEY
+              export AWS_DEFAULT_REGION=$PERFORMANCE_EC2_REGION
+              export EC2_PRIVATE_PEM=$PERFORMANCE_EC2_PRIVATE_PEM
+
+              python3 remote-runner.py \
+                --terraform_bin_path /workspace/terraform \
+                --module_path /workspace/$MODULE_ARTIFACT \
+                --github_actor $CIRCLE_USERNAME \
+                --github_repo $CIRCLE_PROJECT_REPONAME \
+                --github_org $CIRCLE_PROJECT_USERNAME \
+                --github_sha $CIRCLE_SHA1 \
+                --github_branch $CIRCLE_BRANCH \
+                --upload_results_s3 \
+                --triggering_env circleci \
+                --push_results_redistimeseries
+
+
 
 on-any-branch: &on-any-branch
   filters:
@@ -250,14 +321,24 @@ workflows:
     jobs:
       - build:
           <<: *on-any-branch
+      - build_test:
+          <<: *on-any-branch
+          requires:
+            - build
+      - build_memcheck:
+          <<: *on-any-branch
+          requires:
+            - build
       - package_branch:
           <<: *on-master
           requires:
-            - build
+            - build_test
+            - build_memcheck
       - package_release:
           <<: *on-version-tags
           requires:
-            - build
+            - build_test
+            - build_memcheck
       - deploy_branch:
           requires:
             - package_branch
@@ -268,6 +349,10 @@ workflows:
           <<: *on-version-tags
           requires:
             - package_release
+      - performance_ci_automation:
+          <<: *on-any-branch
+          requires:
+            - build_test
 
   nightly:
     triggers:

.gitignore

@@ -52,4 +52,7 @@ deps/GraphBLAS/Config/GraphBLAS.h.tmp
 *.make
 *.xml
 *.iml
-*.d
+*.d
+
+# local/remote benchmark result files
+tests/benchmarks/*.json

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean package docker docker_push docker_alpine builddocs localdocs deploydocs test test_valgrind
+.PHONY: all clean package docker docker_push docker_alpine builddocs localdocs deploydocs test benchmark test_valgrind
 
 all:
 	@$(MAKE) -C ./src all
@@ -36,6 +36,9 @@ deploydocs: builddocs
 test:
 	@$(MAKE) -C ./src test
 
+benchmark:
+	@$(MAKE) -C ./src benchmark
+
 memcheck:
 	@$(MAKE) -C ./src memcheck
 

src/Makefile

@@ -207,3 +207,7 @@ memcheck: CFLAGS += -fno-omit-frame-pointer -g -ggdb -O0 -D MEMCHECK
 memcheck: SHOBJ_LDFLAGS += -u RediSearch_CleanupModule
 memcheck: redisgraph.so
 	@$(MAKE) -C ../tests memcheck
+
+
+benchmark: redisgraph.so
+	@$(MAKE) -C ../tests benchmark

tests/Makefile

@@ -1,7 +1,7 @@
 
 MAKEFLAGS += --no-builtin-rules
 
-.PHONY: test unit flow tck memcheck clean
+.PHONY: test unit flow tck memcheck benchmark clean
 
 TEST_ARGS+=--clear-logs
 
@@ -10,6 +10,10 @@ ifneq ($(V),)
 	TEST_ARGS += --verbose
 endif
 
+ifneq ($(BENCHMARK),)
+	BENCHMARK_ARGS += --test $(BENCHMARK)
+endif
+
 # check environment flag
 ifneq ($(ENV),)
 	TEST_ARGS += --env $(ENV)
@@ -43,6 +47,9 @@ memcheck:
 	@$(MAKE) -C tck TEST_ARGS="$(MEMCHECK_ARGS)"
 	./memcheck.sh
 
+benchmark:
+	cd benchmarks; python3 local-runner.py $(BENCHMARK_ARGS) ; cd ..
+
 clean:
 	@find . -name '*.[oad]' -type f -delete
 	@find . -name '*.run' -type f -delete