Publish utbot ci

.github/workflows/build-utbot.yml

@@ -20,29 +20,24 @@ jobs:
         UTBOT_INSTALL_DIR: /utbot_distr/install
         GRPC_PATH: /utbot_distr/install
         CLI_PATH: /utbot_distr/cli
+        DOCKER_TAG: 20-10-2021
+        ARTIFACT_DIR: utbot-artifact
     steps:
-      - name: Install git
-        run: |
-          apt install -y software-properties-common
-          apt update
-          add-apt-repository -y ppa:git-core/ppa
-          apt update
-          apt install -y git
       - name: Checkout repository
         uses: actions/checkout@v2
         with:
           submodules: recursive
       - name: build UTBot and run unit tests
         run: |
-          source docker/building_dependencies/runtime_env.sh
-          chmod +x build.sh
-          ./build.sh
-          cd server/build
-          chmod +x UTBot_UnitTests
-          ./UTBot_UnitTests info --log `pwd`
+          chmod +x docker/action-scripts/build-and-unit-tests.sh
+          ./docker/action-scripts/build-and-unit-tests.sh
         shell: bash
       - uses: actions/upload-artifact@v2
         if: failure()
         with:
           name: UTBot unit tests logs
-          path: server/build/logs
+          path: server/build/logs
+      - name: build VSCode plugin
+        run: |
+          chmod +x docker/action-scripts/build-vsix.sh
+          ./docker/action-scripts/build-vsix.sh

.github/workflows/publish-utbot.yml

@@ -0,0 +1,51 @@
+name: Publish UTBot as an archive
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  publish:
+    runs-on: ubuntu-18.04
+    env:
+      # Environment for the job and for the container is, sadly, duplicated
+      # https://github.com/docker/compose/issues/1837
+      VERSION: ${{ format('1.0.{0}', github.run_number) }}
+      ARTIFACT_DIR: utbot-artifact
+    container:
+      # It is impossible to parametrize image via workflow environment,
+      # so its name is hard-coded.
+      # https://github.com/actions/runner/issues/480
+      image: ghcr.io/unittestbot/utbotcpp/base_env:20-10-2021
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      env:
+        UTBOT_ALL: /utbot_distr
+        UTBOT_INSTALL_DIR: /utbot_distr/install
+        GRPC_PATH: /utbot_distr/install
+        CLI_PATH: /utbot_distr/cli
+        VERSION: ${{ format('1.0.{0}', github.run_number) }}
+        ARTIFACT_DIR: utbot-artifact
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: build VSCode plugin
+        run: |
+          chmod +x docker/action-scripts/build-vsix.sh
+          ./docker/action-scripts/build-vsix.sh
+      - name: build UTBot
+        run: |
+          chmod +x docker/action-scripts/build-utbot.sh
+          ./docker/action-scripts/build-utbot.sh
+        shell: bash
+      - name: prepare UTBot release for Ubuntu
+        run: |
+          chmod +x docker/action-scripts/prepare-ubuntu-release.sh
+          ./docker/action-scripts/prepare-ubuntu-release.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: utbot-release-${{ env.VERSION }}
+          path: ${{ env.ARTIFACT_DIR }}

.gitmodules

@@ -1,18 +1,18 @@
 [submodule "klee"]
-	path = klee
+	path = submodules/klee
 	url = https://github.com/UnitTestBot/klee.git
 [submodule "Bear"]
-	path = Bear
+	path = submodules/Bear
 	url = https://github.com/UnitTestBot/Bear.git
 [submodule "loguru"]
-	path = loguru
+	path = submodules/loguru
 	url = https://github.com/emilk/loguru.git
 [submodule "json"]
-	path = json
+	path = submodules/json
 	url = https://github.com/astoeckel/json.git
 [submodule "ordered-map"]
-	path = ordered-map
+	path = submodules/ordered-map
 	url = https://github.com/Tessil/ordered-map
 [submodule "parallel-hashmap"]
-	path = parallel-hashmap
+	path = submodules/parallel-hashmap
 	url = https://github.com/greg7mdp/parallel-hashmap

DEVNOTE.md

@@ -23,12 +23,13 @@ To build UTBot from sources:
 
 5. `cd` into `UTBotCpp` directory and run `build.sh` — it is the script that builds KLEE UTBot and runs UTBot unit tests
 6. Navigate to `UTBotCpp/server/build` directory and launch the binary with `./utbot server` command. Now the server is running.
-7. Launch VS Code on your local machine. Use VS Code [Remote-SSH](https://code.visualstudio.com/docs/remote/ssh) to get access to the docker insides. Navigate to `UTBotCpp/vscode-plugin` directory and press F5. This will build and run UTBot VS Code plugin.
-8. A new VS Code window will open; this window will have UTBot VS Code plugin enabled. In this new folder, open `UTBotCpp/integration-tests/c-example` directory.
-9.  When UTBot Quickstart Wizard requests you to enter server host and port, specify `localhost` and UTBot server run port, respectively.
+7. Launch VS Code on your local machine. Use VS Code [Remote-SSH](https://code.visualstudio.com/docs/remote/ssh) to get access to the docker insides. Navigate to `UTBotCpp/vscode-plugin` directory and run `build.sh` script.
+8. Press F5 (*Run Extension*). This will run UTBot VS Code plugin.
+9. A new VS Code window will open; this window will have UTBot VS Code plugin enabled. In this new folder, open `UTBotCpp/integration-tests/c-example` directory.
+10.  When UTBot Quickstart Wizard requests you to enter server host and port, specify `localhost` and UTBot server run port, respectively.
    ![Oops, something went wrong! Please look at wizardInstall.gif](media/wizardInstall.gif "UTBot Wizard Demo")
-6.  Select project path as `/home/utbot/UTBotCpp/integration-tests/c-example`;
-7.  You are now ready to experience UTBot capabilities! You can view possible commands in Command Palette (Press F1 and type in UTBot).
+11.  Select project path as `/home/utbot/UTBotCpp/integration-tests/c-example`;
+12.  You are now ready to experience UTBot capabilities! You can view possible commands in Command Palette (Press F1 and type in UTBot).
 
 If you want to change UTBot test generation preferences, you can edit them in  File > Preferences > Settings > Extensions > UnitTestBot.
 After UTBot configuration, you can select your source directories with the tab on the VSCode toolbar on the left. Then, you can generate tests with the use of Command Palette. Press **F1** and type in "UTBot": You will see tests generation options.

README.md

@@ -1,2 +1,29 @@
-# UTBotCpp
-Tool that generates unit test by C/C++ source code, trying to reach all branches and maximize code coverage
+# What is UTBotCpp?
+UTBotCpp generates test cases by code, trying to cover maximum statements and execution paths. 
+We treat source code as source of truth assuming that behavior is correct and corresponds to initial user demand.
+Generated tests are placed in so-called regression suite.
+Thus, we fixate current behavior by generated test cases.
+Using UTBotCpp developers obtain full control of their code.
+No future change can break the code without being noticed once it's covered with tests generated by UTBot.
+This way, modifications made by developers to an existing code are much safer.
+Hence, with the help of generated unit tests, UTBot provides dramatic code quality improvement.
+
+## How to install and use UTBot
+
+For now, you can only use UTBot under Ubuntu.
+UTBot has been tested under Ubuntu 14, 16, and 18.
+Navigate to the **Releases** GitHub page and download any version of UTBot.
+UTBot is distrbuted as an archive that contains
+1. A pack `utbot_distr.tar.gz` that contains UTBot binary and its dependencies;
+2. UTBot plugin for Visual Studio code — `utbot_plugin.vsix`;
+3. A version `version.txt`;
+4. A run script `unpack_and_run_utbot.sh`.
+
+To launch UTBot, `unzip` the archive and run the 
+`chmod +x unpack_and_run_utbot.sh && ./unpack_and_run_utbot.sh` command (we recommend doing it in a fresh directory to make UTBot removing easier). To remove UTBot, simply delete this directory.
+
+To install UTBot VSCode plugin, use VSCode *Install from VSIX* command.
+
+## How to contribute to UTBot
+
+See **DEVNOTE.md**.

build.sh

@@ -6,7 +6,7 @@
 set -e
 set -o pipefail
 pwd=$PWD
-chmod +x $pwd/klee/build.sh $pwd/Bear/build.sh $pwd/server/build.sh
-cd $pwd/klee && ./build.sh
-cd $pwd/Bear && ./build.sh
+chmod +x $pwd/submodules/klee/build.sh $pwd/submodules/Bear/build.sh $pwd/server/build.sh
+cd $pwd/submodules/klee && ./build.sh
+cd $pwd/submodules/Bear && ./build.sh
 cd $pwd/server && ./build.sh

docker/Dockerfile_base

@@ -127,6 +127,12 @@ RUN echo "StrictHostKeyChecking no" >> /home/utbot/.ssh/config
 
 USER root
 ARG OPERATING_SYSTEM_TAG
+
+# When we switch to Ubuntu 20 container, we may still need old libssl if we support Ubuntu < 18
+# RUN "deb http://security.ubuntu.com/ubuntu bionic-security main" | sudo tee -a /etc/apt/sources.list
+# RUN apt update && apt-cache policy libssl1.0-dev
+# RUN apt install -y --no-install-recommends libssl1.0-dev
+
 RUN if [[ "$OPERATING_SYSTEM_TAG" = "18.04" ]] ; then apt update && apt install -y --no-install-recommends nodejs-dev node-gyp libssl1.0-dev ; fi
 RUN apt update && apt install -y --no-install-recommends nodejs npm openssh-server net-tools gdb vim-nox rsync
 RUN pip3 install git+https://chromium.googlesource.com/external/gyp

docker/action-scripts/build-and-unit-tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+source docker/building_dependencies/runtime_env.sh
+chmod +x docker/action-scripts/build-utbot.sh
+./docker/action-scripts/build-utbot.sh
+cd server/build
+chmod +x UTBot_UnitTests
+./UTBot_UnitTests info --log `pwd`

docker/action-scripts/build-utbot.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+source docker/building_dependencies/runtime_env.sh
+chmod +x build.sh
+./build.sh

docker/action-scripts/build-vsix.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+source docker/building_dependencies/runtime_env.sh
+cd vscode-plugin
+echo $VERSION
+npm version $VERSION --allow-same-version
+utbot_dir="$(dirname $PWD)"
+npm install -g vsce --unsafe
+npm install --unsafe
+npm rebuild grpc --runtime=electron --target=7.3.0
+chmod +x protoc.sh && ./protoc.sh $utbot_dir/server/proto $utbot_dir/vscode-plugin/src/proto-ts
+npm run compile
+vsce package --no-yarn

docker/action-scripts/prepare-ubuntu-release.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+chmod +x docker/building_dependencies/install_packages/debian.sh && ./docker/building_dependencies/install_packages/debian.sh
+chmod +x docker/building_dependencies/clean_release.sh && ./docker/building_dependencies/clean_release.sh
+
+mkdir $ARTIFACT_DIR
+chmod 777 $ARTIFACT_DIR
+
+UTBOT_DISTR_FOLDER=$ARTIFACT_DIR/utbot_distr
+if [ -d "$UTBOT_DISTR_FOLDER" ]; then rm -rf $UTBOT_DISTR_FOLDER; fi
+
+cp -r /utbot_distr $UTBOT_DISTR_FOLDER
+cp -a docker/release_distribution_scripts/. $UTBOT_DISTR_FOLDER/
+echo $VERSION > $ARTIFACT_DIR/version.txt
+
+mv vscode-plugin/*.vsix $ARTIFACT_DIR/utbot_plugin.vsix
+mv docker/unpack_and_run_utbot.sh $ARTIFACT_DIR/unpack_and_run_utbot.sh
+
+cd $ARTIFACT_DIR
+tar -czf utbot_distr.tar.gz utbot_distr
+rm -rf utbot_distr

docker/building_dependencies/clean_release.sh

@@ -0,0 +1,41 @@
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+# This script clears out all the unnecessary files from the release build
+
+# Need just clang-10, llvm-cov, llvm-profdata and cmake in $UTBOT_INSTALL_DIR/bin
+cd $UTBOT_INSTALL_DIR/bin && find -type f \( -name "*" ! -name "llvm-cov" ! -name "llvm-profdata"  ! -name "llvm-nm" ! -name "clang" ! -name "clang++" ! -name "clang-10" ! -name cmake \) -delete
+
+# Delete all broken links. Don't need to delete clang which is a link (not broken) to clang-10
+find $UTBOT_INSTALL_DIR/bin/ -xtype l -delete
+
+# Delete all regular files in $UTBOT_INSTALL_DIR/lib except gold plugin, but not recursively
+cd $UTBOT_INSTALL_DIR/lib && find -maxdepth 1 -type f \( -name "*" ! -name "LLVMgold.so" ! -name "libz3.so*" \) -delete
+
+#TODO some libs are needed:
+# Only certain libs should remain in $UTBOT_INSTALL_DIR/lib/clang/10.0.0/lib/linux
+#cd $UTBOT_INSTALL_DIR/lib/clang/10.0.0/lib/linux &&  find -maxdepth 1 -type f \( -name "*" ! -name "libclang_rt.ubsan_standalone-x86_64.a" ! -name "libclang_rt.ubsan_standalone_cxx-x86_64.a" ! -name "libclang_rt.profile-x86_64.a" \) -delete
+
+rm -rf $UTBOT_ALL/klee/bin
+
+# There should be just some klee libraries in klee/lib
+cd $UTBOT_ALL/klee/lib && find -type f \( -name "*" ! -name "libkleeRuntimeFp64_Debug+Asserts.bca" ! -name "libkleeRuntimeFp64_Debug+Asserts.bca" ! -name "libkleeRuntimeKLEELibc64_Debug+Asserts.bca" ! -name "libkleeRuntimeKLEELibc64_Debug+Asserts.bca" ! -name "libkleeRuntimeFreestanding64_Debug+Asserts.bca" ! -name "libkleeRuntimeIntrinsic64_Debug+Asserts.bca" ! -name "libkleeRuntimePOSIX64_Debug+Asserts.bca" ! -name "klee-uclibc.bca" \) -delete
+
+# Previous command removed only regular files, need to remove also symlincs and then empty directories
+find $UTBOT_ALL/klee -type l -delete
+find $UTBOT_ALL/klee -type d -empty -delete
+
+rm -rf $UTBOT_ALL/tsl
+rm -rf $UTBOT_ALL/parallel_hashmap
+rm -rf $UTBOT_ALL/cli
+rm -rf $UTBOT_ALL/json
+# we use klee libc
+rm -rf $UTBOT_ALL/uclibc/
+rm -rf $UTBOT_ALL/node_modules
+
+# there should be error, it's okay, directory remains with some files
+# dpkg should remain because it is useful for apt update
+mv $UTBOT_ALL/debs-install/usr/share/dpkg ~ && rm -rf $UTBOT_ALL/debs-install/usr/share &&  mv ~/dpkg $UTBOT_ALL/debs-install/usr/share
+
+rm -rf $UTBOT_INSTALL_DIR/include

docker/building_dependencies/install_packages/debian.sh

@@ -0,0 +1,58 @@
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+# This script downloads debian packages.
+# It installs libc6-dev package (which is dev version of libc) into $UTBOT_ALL/debian-libc-dev-install and other packages into $UTBOT_ALL/debs-install
+
+set -e
+set -o pipefail
+
+# Downloading apt-rdepends tool which can get all the dependencies for a package
+apt-get update && apt-get  -y --no-install-recommends install apt-rdepends && apt-get update
+rm -rf /tmp/debian_dependencies && mkdir -p /tmp/debian_dependencies && cd /tmp/debian_dependencies
+# expand_aliases is used to support alias command properly in bash script
+shopt -s expand_aliases
+# A grep command which clears out the output of apt-rdepends
+alias grepdepends='grep -v "^ " | grep -v "^libc-dev$" | grep -v "^debconf-2.0$" | grep -v "^libc6$" | grep -v "^libunwind8-dev$" | grep -v "^awk$"'
+# Get all the dependencies of utbot
+apt-rdepends libsqlite3-dev libgoogle-perftools-dev libssl-dev libssl1.0-dev python3-pip gzip make gcc-9 g++-9 | grepdepends > all.txt
+# Get all the dependencies of libc6-dev
+apt-rdepends libc6-dev | grepdepends > debian-libc-dev.txt
+# Get all the dependencies of utbot except all the dependencies of libc6-dev
+diff --new-line-format="" --unchanged-line-format="" <(sort all.txt) <(sort debian-libc-dev.txt) > all_without_libc-dev.txt || :
+
+# A function which downloads all the dependencies from text file and extracts them.
+# Prerequisites: file path/to/file.txt exists
+# Arguments:
+#   $1 = path/to/file  The first argument is a path to a file without the .txt extension
+# The extracted packages will be located in $UTBOT_ALL/path/to/file-install directory
+get_debian_packages() {
+  # Create a directory for .deb packages
+  rm -rf $1 && mkdir -p $1 && cd $1
+  # Download .deb packages
+  apt-get download $(cat ../$1.txt)
+  cd ..
+
+  # Extract all the .deb packages into $UTBOT_ALL/$1-install directory
+  for filename in $1/*.deb; do
+          dpkg-deb -x "$filename" $UTBOT_ALL/$1-install
+  done
+}
+
+# Get all packages except libc6-dev
+get_debian_packages all_without_libc-dev
+rm -rf $UTBOT_ALL/debs-install
+mv $UTBOT_ALL/all_without_libc-dev-install $UTBOT_ALL/debs-install
+# Get libc6-dev package and it's dependencies
+get_debian_packages debian-libc-dev
+
+# Creating links to the current versions of gcc and python
+cd $UTBOT_ALL/debs-install/usr/bin
+ln -s python3 python
+ln -s gcov-9 gcov
+ln -s g++-9 g++
+ln -s gcc-9 gcc
+
+# Setup python packages
+pip3 install --target=$UTBOT_ALL/debs-install/usr/local/lib/python3.4/dist-packages/ --ignore-installed typing

docker/release_distribution_scripts/utbot_online_cli.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+#
+# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved.
+#
+
+# This script launches cli for UTBot online
+#arguments: (generate | run) (path to project) (path to snippet)
+
+if [ "$#" -ne 3 ] || { [ "$1" != "generate" ] && [ "$1" != "run" ]; };
+then
+    echo "Illegal number of parameters. Check: (generate | run) (path-project) (path-snippet)"
+    exit 1;
+fi
+
+# Identify the directory where the current script is located
+export CURRENT_FOLDER="$( cd $( dirname ${BASH_SOURCE[0]} ) && pwd )"
+# Get full path a script that launches UTBot
+RUN_SYSTEM_SCRIPT_PATH=$CURRENT_FOLDER/utbot_run_system.sh
+UTBOT_CLI_OPTIONS="$1 --project-path $2 file --file-path $3"
+# Launching cli
+$RUN_SYSTEM_SCRIPT_PATH "cli" $UTBOT_CLI_OPTIONS