diff --git a/.github/workflows/build-utbot.yml b/.github/workflows/build-utbot.yml index 64b73fc53..e443e8536 100644 --- a/.github/workflows/build-utbot.yml +++ b/.github/workflows/build-utbot.yml @@ -20,29 +20,24 @@ jobs: UTBOT_INSTALL_DIR: /utbot_distr/install GRPC_PATH: /utbot_distr/install CLI_PATH: /utbot_distr/cli + DOCKER_TAG: 20-10-2021 + ARTIFACT_DIR: utbot-artifact steps: - - name: Install git - run: | - apt install -y software-properties-common - apt update - add-apt-repository -y ppa:git-core/ppa - apt update - apt install -y git - name: Checkout repository uses: actions/checkout@v2 with: submodules: recursive - name: build UTBot and run unit tests run: | - source docker/building_dependencies/runtime_env.sh - chmod +x build.sh - ./build.sh - cd server/build - chmod +x UTBot_UnitTests - ./UTBot_UnitTests info --log `pwd` + chmod +x docker/action-scripts/build-and-unit-tests.sh + ./docker/action-scripts/build-and-unit-tests.sh shell: bash - uses: actions/upload-artifact@v2 if: failure() with: name: UTBot unit tests logs - path: server/build/logs \ No newline at end of file + path: server/build/logs + - name: build VSCode plugin + run: | + chmod +x docker/action-scripts/build-vsix.sh + ./docker/action-scripts/build-vsix.sh \ No newline at end of file diff --git a/.github/workflows/publish-utbot.yml b/.github/workflows/publish-utbot.yml new file mode 100644 index 000000000..90544d353 --- /dev/null +++ b/.github/workflows/publish-utbot.yml @@ -0,0 +1,51 @@ +name: Publish UTBot as an archive + +on: + push: + branches: [main] + +jobs: + publish: + runs-on: ubuntu-18.04 + env: + # Environment for the job and for the container is, sadly, duplicated + # https://github.com/docker/compose/issues/1837 + VERSION: ${{ format('1.0.{0}', github.run_number) }} + ARTIFACT_DIR: utbot-artifact + container: + # It is impossible to parametrize image via workflow environment, + # so its name is hard-coded. + # https://github.com/actions/runner/issues/480 + image: ghcr.io/unittestbot/utbotcpp/base_env:20-10-2021 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + env: + UTBOT_ALL: /utbot_distr + UTBOT_INSTALL_DIR: /utbot_distr/install + GRPC_PATH: /utbot_distr/install + CLI_PATH: /utbot_distr/cli + VERSION: ${{ format('1.0.{0}', github.run_number) }} + ARTIFACT_DIR: utbot-artifact + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + - name: build VSCode plugin + run: | + chmod +x docker/action-scripts/build-vsix.sh + ./docker/action-scripts/build-vsix.sh + - name: build UTBot + run: | + chmod +x docker/action-scripts/build-utbot.sh + ./docker/action-scripts/build-utbot.sh + shell: bash + - name: prepare UTBot release for Ubuntu + run: | + chmod +x docker/action-scripts/prepare-ubuntu-release.sh + ./docker/action-scripts/prepare-ubuntu-release.sh + - uses: actions/upload-artifact@v2 + with: + name: utbot-release-${{ env.VERSION }} + path: ${{ env.ARTIFACT_DIR }} \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index bc0b423a6..ec3f420a3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,18 @@ [submodule "klee"] - path = klee + path = submodules/klee url = https://github.com/UnitTestBot/klee.git [submodule "Bear"] - path = Bear + path = submodules/Bear url = https://github.com/UnitTestBot/Bear.git [submodule "loguru"] - path = loguru + path = submodules/loguru url = https://github.com/emilk/loguru.git [submodule "json"] - path = json + path = submodules/json url = https://github.com/astoeckel/json.git [submodule "ordered-map"] - path = ordered-map + path = submodules/ordered-map url = https://github.com/Tessil/ordered-map [submodule "parallel-hashmap"] - path = parallel-hashmap + path = submodules/parallel-hashmap url = https://github.com/greg7mdp/parallel-hashmap diff --git a/DEVNOTE.md b/DEVNOTE.md index b6e8b6a90..838482ee2 100644 --- a/DEVNOTE.md +++ b/DEVNOTE.md @@ -23,12 +23,13 @@ To build UTBot from sources: 5. `cd` into `UTBotCpp` directory and run `build.sh` — it is the script that builds KLEE UTBot and runs UTBot unit tests 6. Navigate to `UTBotCpp/server/build` directory and launch the binary with `./utbot server` command. Now the server is running. -7. Launch VS Code on your local machine. Use VS Code [Remote-SSH](https://code.visualstudio.com/docs/remote/ssh) to get access to the docker insides. Navigate to `UTBotCpp/vscode-plugin` directory and press F5. This will build and run UTBot VS Code plugin. -8. A new VS Code window will open; this window will have UTBot VS Code plugin enabled. In this new folder, open `UTBotCpp/integration-tests/c-example` directory. -9. When UTBot Quickstart Wizard requests you to enter server host and port, specify `localhost` and UTBot server run port, respectively. +7. Launch VS Code on your local machine. Use VS Code [Remote-SSH](https://code.visualstudio.com/docs/remote/ssh) to get access to the docker insides. Navigate to `UTBotCpp/vscode-plugin` directory and run `build.sh` script. +8. Press F5 (*Run Extension*). This will run UTBot VS Code plugin. +9. A new VS Code window will open; this window will have UTBot VS Code plugin enabled. In this new folder, open `UTBotCpp/integration-tests/c-example` directory. +10. When UTBot Quickstart Wizard requests you to enter server host and port, specify `localhost` and UTBot server run port, respectively. ![Oops, something went wrong! Please look at wizardInstall.gif](media/wizardInstall.gif "UTBot Wizard Demo") -6. Select project path as `/home/utbot/UTBotCpp/integration-tests/c-example`; -7. You are now ready to experience UTBot capabilities! You can view possible commands in Command Palette (Press F1 and type in UTBot). +11. Select project path as `/home/utbot/UTBotCpp/integration-tests/c-example`; +12. You are now ready to experience UTBot capabilities! You can view possible commands in Command Palette (Press F1 and type in UTBot). If you want to change UTBot test generation preferences, you can edit them in File > Preferences > Settings > Extensions > UnitTestBot. After UTBot configuration, you can select your source directories with the tab on the VSCode toolbar on the left. Then, you can generate tests with the use of Command Palette. Press **F1** and type in "UTBot": You will see tests generation options. diff --git a/README.md b/README.md index ab26e5272..89cd4d061 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,29 @@ -# UTBotCpp -Tool that generates unit test by C/C++ source code, trying to reach all branches and maximize code coverage +# What is UTBotCpp? +UTBotCpp generates test cases by code, trying to cover maximum statements and execution paths. +We treat source code as source of truth assuming that behavior is correct and corresponds to initial user demand. +Generated tests are placed in so-called regression suite. +Thus, we fixate current behavior by generated test cases. +Using UTBotCpp developers obtain full control of their code. +No future change can break the code without being noticed once it's covered with tests generated by UTBot. +This way, modifications made by developers to an existing code are much safer. +Hence, with the help of generated unit tests, UTBot provides dramatic code quality improvement. + +## How to install and use UTBot + +For now, you can only use UTBot under Ubuntu. +UTBot has been tested under Ubuntu 14, 16, and 18. +Navigate to the **Releases** GitHub page and download any version of UTBot. +UTBot is distrbuted as an archive that contains +1. A pack `utbot_distr.tar.gz` that contains UTBot binary and its dependencies; +2. UTBot plugin for Visual Studio code — `utbot_plugin.vsix`; +3. A version `version.txt`; +4. A run script `unpack_and_run_utbot.sh`. + +To launch UTBot, `unzip` the archive and run the +`chmod +x unpack_and_run_utbot.sh && ./unpack_and_run_utbot.sh` command (we recommend doing it in a fresh directory to make UTBot removing easier). To remove UTBot, simply delete this directory. + +To install UTBot VSCode plugin, use VSCode *Install from VSIX* command. + +## How to contribute to UTBot + +See **DEVNOTE.md**. \ No newline at end of file diff --git a/build.sh b/build.sh index 318c8def1..5cc817e54 100644 --- a/build.sh +++ b/build.sh @@ -6,7 +6,7 @@ set -e set -o pipefail pwd=$PWD -chmod +x $pwd/klee/build.sh $pwd/Bear/build.sh $pwd/server/build.sh -cd $pwd/klee && ./build.sh -cd $pwd/Bear && ./build.sh +chmod +x $pwd/submodules/klee/build.sh $pwd/submodules/Bear/build.sh $pwd/server/build.sh +cd $pwd/submodules/klee && ./build.sh +cd $pwd/submodules/Bear && ./build.sh cd $pwd/server && ./build.sh diff --git a/docker/Dockerfile_base b/docker/Dockerfile_base index 5b0bb9f81..c71e3e165 100644 --- a/docker/Dockerfile_base +++ b/docker/Dockerfile_base @@ -127,6 +127,12 @@ RUN echo "StrictHostKeyChecking no" >> /home/utbot/.ssh/config USER root ARG OPERATING_SYSTEM_TAG + +# When we switch to Ubuntu 20 container, we may still need old libssl if we support Ubuntu < 18 +# RUN "deb http://security.ubuntu.com/ubuntu bionic-security main" | sudo tee -a /etc/apt/sources.list +# RUN apt update && apt-cache policy libssl1.0-dev +# RUN apt install -y --no-install-recommends libssl1.0-dev + RUN if [[ "$OPERATING_SYSTEM_TAG" = "18.04" ]] ; then apt update && apt install -y --no-install-recommends nodejs-dev node-gyp libssl1.0-dev ; fi RUN apt update && apt install -y --no-install-recommends nodejs npm openssh-server net-tools gdb vim-nox rsync RUN pip3 install git+https://chromium.googlesource.com/external/gyp diff --git a/docker/action-scripts/build-and-unit-tests.sh b/docker/action-scripts/build-and-unit-tests.sh new file mode 100644 index 000000000..0343fabf6 --- /dev/null +++ b/docker/action-scripts/build-and-unit-tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +source docker/building_dependencies/runtime_env.sh +chmod +x docker/action-scripts/build-utbot.sh +./docker/action-scripts/build-utbot.sh +cd server/build +chmod +x UTBot_UnitTests +./UTBot_UnitTests info --log `pwd` \ No newline at end of file diff --git a/docker/action-scripts/build-utbot.sh b/docker/action-scripts/build-utbot.sh new file mode 100644 index 000000000..45cde3583 --- /dev/null +++ b/docker/action-scripts/build-utbot.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +source docker/building_dependencies/runtime_env.sh +chmod +x build.sh +./build.sh diff --git a/docker/action-scripts/build-vsix.sh b/docker/action-scripts/build-vsix.sh new file mode 100644 index 000000000..d07455330 --- /dev/null +++ b/docker/action-scripts/build-vsix.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +source docker/building_dependencies/runtime_env.sh +cd vscode-plugin +echo $VERSION +npm version $VERSION --allow-same-version +utbot_dir="$(dirname $PWD)" +npm install -g vsce --unsafe +npm install --unsafe +npm rebuild grpc --runtime=electron --target=7.3.0 +chmod +x protoc.sh && ./protoc.sh $utbot_dir/server/proto $utbot_dir/vscode-plugin/src/proto-ts +npm run compile +vsce package --no-yarn \ No newline at end of file diff --git a/docker/action-scripts/prepare-ubuntu-release.sh b/docker/action-scripts/prepare-ubuntu-release.sh new file mode 100644 index 000000000..be125a395 --- /dev/null +++ b/docker/action-scripts/prepare-ubuntu-release.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +chmod +x docker/building_dependencies/install_packages/debian.sh && ./docker/building_dependencies/install_packages/debian.sh +chmod +x docker/building_dependencies/clean_release.sh && ./docker/building_dependencies/clean_release.sh + +mkdir $ARTIFACT_DIR +chmod 777 $ARTIFACT_DIR + +UTBOT_DISTR_FOLDER=$ARTIFACT_DIR/utbot_distr +if [ -d "$UTBOT_DISTR_FOLDER" ]; then rm -rf $UTBOT_DISTR_FOLDER; fi + +cp -r /utbot_distr $UTBOT_DISTR_FOLDER +cp -a docker/release_distribution_scripts/. $UTBOT_DISTR_FOLDER/ +echo $VERSION > $ARTIFACT_DIR/version.txt + +mv vscode-plugin/*.vsix $ARTIFACT_DIR/utbot_plugin.vsix +mv docker/unpack_and_run_utbot.sh $ARTIFACT_DIR/unpack_and_run_utbot.sh + +cd $ARTIFACT_DIR +tar -czf utbot_distr.tar.gz utbot_distr +rm -rf utbot_distr diff --git a/docker/building_dependencies/clean_release.sh b/docker/building_dependencies/clean_release.sh new file mode 100644 index 000000000..ebe17b4ca --- /dev/null +++ b/docker/building_dependencies/clean_release.sh @@ -0,0 +1,41 @@ +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# This script clears out all the unnecessary files from the release build + +# Need just clang-10, llvm-cov, llvm-profdata and cmake in $UTBOT_INSTALL_DIR/bin +cd $UTBOT_INSTALL_DIR/bin && find -type f \( -name "*" ! -name "llvm-cov" ! -name "llvm-profdata" ! -name "llvm-nm" ! -name "clang" ! -name "clang++" ! -name "clang-10" ! -name cmake \) -delete + +# Delete all broken links. Don't need to delete clang which is a link (not broken) to clang-10 +find $UTBOT_INSTALL_DIR/bin/ -xtype l -delete + +# Delete all regular files in $UTBOT_INSTALL_DIR/lib except gold plugin, but not recursively +cd $UTBOT_INSTALL_DIR/lib && find -maxdepth 1 -type f \( -name "*" ! -name "LLVMgold.so" ! -name "libz3.so*" \) -delete + +#TODO some libs are needed: +# Only certain libs should remain in $UTBOT_INSTALL_DIR/lib/clang/10.0.0/lib/linux +#cd $UTBOT_INSTALL_DIR/lib/clang/10.0.0/lib/linux && find -maxdepth 1 -type f \( -name "*" ! -name "libclang_rt.ubsan_standalone-x86_64.a" ! -name "libclang_rt.ubsan_standalone_cxx-x86_64.a" ! -name "libclang_rt.profile-x86_64.a" \) -delete + +rm -rf $UTBOT_ALL/klee/bin + +# There should be just some klee libraries in klee/lib +cd $UTBOT_ALL/klee/lib && find -type f \( -name "*" ! -name "libkleeRuntimeFp64_Debug+Asserts.bca" ! -name "libkleeRuntimeFp64_Debug+Asserts.bca" ! -name "libkleeRuntimeKLEELibc64_Debug+Asserts.bca" ! -name "libkleeRuntimeKLEELibc64_Debug+Asserts.bca" ! -name "libkleeRuntimeFreestanding64_Debug+Asserts.bca" ! -name "libkleeRuntimeIntrinsic64_Debug+Asserts.bca" ! -name "libkleeRuntimePOSIX64_Debug+Asserts.bca" ! -name "klee-uclibc.bca" \) -delete + +# Previous command removed only regular files, need to remove also symlincs and then empty directories +find $UTBOT_ALL/klee -type l -delete +find $UTBOT_ALL/klee -type d -empty -delete + +rm -rf $UTBOT_ALL/tsl +rm -rf $UTBOT_ALL/parallel_hashmap +rm -rf $UTBOT_ALL/cli +rm -rf $UTBOT_ALL/json +# we use klee libc +rm -rf $UTBOT_ALL/uclibc/ +rm -rf $UTBOT_ALL/node_modules + +# there should be error, it's okay, directory remains with some files +# dpkg should remain because it is useful for apt update +mv $UTBOT_ALL/debs-install/usr/share/dpkg ~ && rm -rf $UTBOT_ALL/debs-install/usr/share && mv ~/dpkg $UTBOT_ALL/debs-install/usr/share + +rm -rf $UTBOT_INSTALL_DIR/include \ No newline at end of file diff --git a/docker/building_dependencies/install_packages/debian.sh b/docker/building_dependencies/install_packages/debian.sh new file mode 100644 index 000000000..1033a5577 --- /dev/null +++ b/docker/building_dependencies/install_packages/debian.sh @@ -0,0 +1,58 @@ +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# This script downloads debian packages. +# It installs libc6-dev package (which is dev version of libc) into $UTBOT_ALL/debian-libc-dev-install and other packages into $UTBOT_ALL/debs-install + +set -e +set -o pipefail + +# Downloading apt-rdepends tool which can get all the dependencies for a package +apt-get update && apt-get -y --no-install-recommends install apt-rdepends && apt-get update +rm -rf /tmp/debian_dependencies && mkdir -p /tmp/debian_dependencies && cd /tmp/debian_dependencies +# expand_aliases is used to support alias command properly in bash script +shopt -s expand_aliases +# A grep command which clears out the output of apt-rdepends +alias grepdepends='grep -v "^ " | grep -v "^libc-dev$" | grep -v "^debconf-2.0$" | grep -v "^libc6$" | grep -v "^libunwind8-dev$" | grep -v "^awk$"' +# Get all the dependencies of utbot +apt-rdepends libsqlite3-dev libgoogle-perftools-dev libssl-dev libssl1.0-dev python3-pip gzip make gcc-9 g++-9 | grepdepends > all.txt +# Get all the dependencies of libc6-dev +apt-rdepends libc6-dev | grepdepends > debian-libc-dev.txt +# Get all the dependencies of utbot except all the dependencies of libc6-dev +diff --new-line-format="" --unchanged-line-format="" <(sort all.txt) <(sort debian-libc-dev.txt) > all_without_libc-dev.txt || : + +# A function which downloads all the dependencies from text file and extracts them. +# Prerequisites: file path/to/file.txt exists +# Arguments: +# $1 = path/to/file The first argument is a path to a file without the .txt extension +# The extracted packages will be located in $UTBOT_ALL/path/to/file-install directory +get_debian_packages() { + # Create a directory for .deb packages + rm -rf $1 && mkdir -p $1 && cd $1 + # Download .deb packages + apt-get download $(cat ../$1.txt) + cd .. + + # Extract all the .deb packages into $UTBOT_ALL/$1-install directory + for filename in $1/*.deb; do + dpkg-deb -x "$filename" $UTBOT_ALL/$1-install + done +} + +# Get all packages except libc6-dev +get_debian_packages all_without_libc-dev +rm -rf $UTBOT_ALL/debs-install +mv $UTBOT_ALL/all_without_libc-dev-install $UTBOT_ALL/debs-install +# Get libc6-dev package and it's dependencies +get_debian_packages debian-libc-dev + +# Creating links to the current versions of gcc and python +cd $UTBOT_ALL/debs-install/usr/bin +ln -s python3 python +ln -s gcov-9 gcov +ln -s g++-9 g++ +ln -s gcc-9 gcc + +# Setup python packages +pip3 install --target=$UTBOT_ALL/debs-install/usr/local/lib/python3.4/dist-packages/ --ignore-installed typing diff --git a/docker/release_distribution_scripts/utbot_online_cli.sh b/docker/release_distribution_scripts/utbot_online_cli.sh new file mode 100644 index 000000000..be79ebdf9 --- /dev/null +++ b/docker/release_distribution_scripts/utbot_online_cli.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# This script launches cli for UTBot online +#arguments: (generate | run) (path to project) (path to snippet) + +if [ "$#" -ne 3 ] || { [ "$1" != "generate" ] && [ "$1" != "run" ]; }; +then + echo "Illegal number of parameters. Check: (generate | run) (path-project) (path-snippet)" + exit 1; +fi + +# Identify the directory where the current script is located +export CURRENT_FOLDER="$( cd $( dirname ${BASH_SOURCE[0]} ) && pwd )" +# Get full path a script that launches UTBot +RUN_SYSTEM_SCRIPT_PATH=$CURRENT_FOLDER/utbot_run_system.sh +UTBOT_CLI_OPTIONS="$1 --project-path $2 file --file-path $3" +# Launching cli +$RUN_SYSTEM_SCRIPT_PATH "cli" $UTBOT_CLI_OPTIONS \ No newline at end of file diff --git a/docker/release_distribution_scripts/utbot_run_system.sh b/docker/release_distribution_scripts/utbot_run_system.sh new file mode 100644 index 000000000..2551225f2 --- /dev/null +++ b/docker/release_distribution_scripts/utbot_run_system.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# This script can launch server, cli and tests +#arguments - add MODE (server | cli | test) + +# Check if arguments are correct +if [ "$1" != "cli" ] && [ "$1" != "server" ] && [ "$1" != "test" ] +then + echo "Wrong UTBOT_MODE: expected cli|server|test" + exit 1 +fi + +#set UTBot release flag +export UTBOT_RELEASE=true + +# Retrieving path to $UTBOT_ALL from absolute path to current script +export UTBOT_ALL=$CURRENT_FOLDER + +# Setting environment variables according to $UTBOT_ALL +export UTBOT_INSTALL_DIR=$UTBOT_ALL/install +export CC=$UTBOT_ALL/debs-install/usr/bin/gcc-9 +export CXX=$UTBOT_ALL/debs-install/usr/bin/g++-9 +export CPATH=$UTBOT_ALL/klee/include:$CPATH # Path for C and C++ includes +export PATH=$UTBOT_ALL/bear/bin:$UTBOT_ALL/klee/bin:$UTBOT_INSTALL_DIR/bin:$PATH +export KLEE_RUNTIME_LIBRARY_PATH=$UTBOT_ALL/klee/lib/klee/runtime/ + +# If the system is opensuse, variable is not empty. It is empty otherwise. +IS_SUSE="$(grep '^NAME=' /etc/os-release | tr '[:upper:]' '[:lower:]' | grep suse)" + +# Setting environment variables for debian packages +export PATH=$UTBOT_ALL/debs-install/usr/bin:$PATH +export LD_LIBRARY_PATH=$UTBOT_ALL/debs-install/usr/lib/x86_64-linux-gnu:$UTBOT_ALL/debs-install/lib/x86_64-linux-gnu:$UTBOT_ALL/debs-install/usr/lib:$UTBOT_ALL/install/lib +export CPATH=$UTBOT_ALL/debs-install/usr/include:$UTBOT_ALL/debs-install/usr/include/x86_64-linux-gnu/:$CPATH +export C_INCLUDE_PATH=$UTBOT_ALL/debs-install/usr/include:$UTBOT_ALL/debs-install/usr/lib/gcc/x86_64-linux-gnu/9/include/:$C_INCLUDE_PATH +export CPLUS_INCLUDE_PATH=$UTBOT_ALL/debs-install/usr/include/c++/9:$UTBOT_ALL/debs-install/usr/include/x86_64-linux-gnu/c++/9:$UTBOT_ALL/debs-install/usr/include/c++/9/backward:$UTBOT_ALL/debs-install/usr/include +export LDFLAGS="-fuse-ld=gold -B $UTBOT_ALL/debs-install/usr/lib/gcc/x86_64-linux-gnu/9/ -L $UTBOT_ALL/debs-install/usr/lib/gcc/x86_64-linux-gnu/9/ -B $UTBOT_ALL/debs-install/usr/lib/x86_64-linux-gnu/ -L $UTBOT_ALL/debs-install/usr/lib/x86_64-linux-gnu/ -L$UTBOT_ALL/debs-install/usr/lib64/ -B $UTBOT_ALL/debs-install/usr/lib64/ -L /lib64/ -B /lib64/" # Paths for object files and libraries with which compiler should link the project +# This function moves dev version of libc into $UTBOT_ALL/debs-install directory +# Prerequisites: path/to/directory should exist +# Arguments: +# $1 = path/to/directory The first argument is a path to a directory dev libc package +move-libc-dev() { + # If dev libc has already been moved, skipping + if test -e "$UTBOT_ALL/$1"; then + cp -r $UTBOT_ALL/$1/* $UTBOT_ALL/debs-install + rm -rf $UTBOT_ALL/$1 + fi +} + +if [ -z "$IS_SUSE" ] +then + # If the system is not suse, use debian packages + move-libc-dev debian-libc-dev-install + export X86_LIBS=lib/x86_64-linux-gnu +else + # If the system is suse, use rpm packages + move-libc-dev suse-libc-dev-install + export X86_LIBS=lib64 +fi +# Updating libm.so so that it contains valid path to libmvec_nonshared.a +echo "/* GNU ld script */ +OUTPUT_FORMAT(elf64-x86-64) +GROUP ( /$X86_LIBS/libm.so.6 AS_NEEDED ( $UTBOT_ALL/debs-install/usr/$X86_LIBS/libmvec_nonshared.a /$X86_LIBS/libmvec.so.1 ) )" > $UTBOT_ALL/debs-install/usr/$X86_LIBS/libm.so + +export LDFLAGS="$LDFLAGS -L$UTBOT_ALL/debs-install/usr/lib64/ -B $UTBOT_ALL/debs-install/usr/lib64/ -L /lib64/ -B /lib64/" + +# Updating libc.so so that it contains valid path to libc_nonshared.a +echo "/* GNU ld script + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ +OUTPUT_FORMAT(elf64-x86-64) +GROUP ( /$X86_LIBS/libc.so.6 $UTBOT_ALL/debs-install/usr/$X86_LIBS/libc_nonshared.a AS_NEEDED ( /$X86_LIBS/ld-linux-x86-64.so.2 ) )" > $UTBOT_ALL/debs-install/usr/$X86_LIBS/libc.so + +# Updating libpthread.so so that it contains valid path to libpthread_nonshared.a +echo "/* GNU ld script + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ +OUTPUT_FORMAT(elf64-x86-64) +GROUP ( /$X86_LIBS/libpthread.so.0 $UTBOT_ALL/debs-install/usr/$X86_LIBS/libpthread_nonshared.a )" > $UTBOT_ALL/debs-install/usr/$X86_LIBS/libpthread.so + +# Creating logs directories so that watchdog and utbot can launch +mkdir -p /home/$USER/logs/watchdog +mkdir -p /home/$USER/logs/utbot + +# Path to common functions +WATCHDOG_SCRIPT_FOLDER=$UTBOT_ALL/utbot_scripts +COMMON_FUNCTIONS_SCRIPT_PATH=$WATCHDOG_SCRIPT_FOLDER/common_functions.sh +source $COMMON_FUNCTIONS_SCRIPT_PATH + +if [ "$1" = "server" ] +then + UTBOT_MODE=server + if [ -z "$2" ] + then + export UTBOT_PORT=2121 + else + export UTBOT_PORT=$2 + fi + + #Server-specific parameters + UTBOT_EXECUTABLE_PATH=$UTBOT_BINARIES_FOLDER/$UTBOT_PROCESS_PATTERN + UTBOT_STDOUT_LOG_FILE=$UTBOT_LOGS_FOLDER/$UTBOT_PROCESS_PATTERN-$(now).log + UTBOT_TMP_FOLDER=$UTBOT_LOGS_FOLDER/tmp + UTBOT_SERVER_OPTIONS="$UTBOT_MODE --port $UTBOT_PORT --log=$UTBOT_LOGS_FOLDER --tmp=$UTBOT_TMP_FOLDER" + + log "Starting a new server process; logs are written into [$UTBOT_LOGS_FOLDER] folder" + start_process $UTBOT_PROCESS_PATTERN $UTBOT_EXECUTABLE_PATH "$UTBOT_SERVER_OPTIONS" $UTBOT_STDOUT_LOG_FILE $UTBOT_PID_FILE +fi + +if [ "$1" = "cli" ] +then + #Online-cli-specific parameters + UTBOT_EXECUTABLE_PATH=$UTBOT_BINARIES_FOLDER/$UTBOT_PROCESS_PATTERN + UTBOT_CLI_OPTIONS="${@:2}" + + if [ "$2" == "generate" ] + then + PROJECT_PATH=$4 + mkdir -p $PROJECT_PATH/build + cd $PROJECT_PATH/build || exit + cmake .. + bear make + cd $CURRENT_FOLDER || exit + fi + + log "Run utbot-cli" + trap 'catch $? $LINENO' ERR + catch() { + echo "Error $1 occurred on $2" + exit 1 + } + $UTBOT_EXECUTABLE_PATH $UTBOT_CLI_OPTIONS + exit 0; +fi + +if [ "$1" = "test" ] +then + TESTS_EXECUTABLE_PATH=./UTBot_UnitTests + + log "Run tests for utbot" + trap 'catch $? $LINENO' ERR + catch() { + echo "Error $1 occurred on $2" + exit 1 + } + cd $UTBOT_BINARIES_FOLDER + $TESTS_EXECUTABLE_PATH $2 $3 + exit 0; +fi diff --git a/docker/release_distribution_scripts/utbot_scripts/common_functions.sh b/docker/release_distribution_scripts/utbot_scripts/common_functions.sh new file mode 100644 index 000000000..35d421e35 --- /dev/null +++ b/docker/release_distribution_scripts/utbot_scripts/common_functions.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# The script provides bash functions that are commonly used in other scripts + +# Common variables +export UTBOT_PROCESS_PATTERN=utbot +export UTBOT_BINARIES_FOLDER=$UTBOT_ALL/server-install +export UTBOT_PID_FILE=$UTBOT_BINARIES_FOLDER/$UTBOT_PROCESS_PATTERN.pid +export UTBOT_LOGS_FOLDER=/home/$USER/logs/utbot + +# Common functions + +# Returns current timestamp +now(){ + echo $(date -d "today" +"%Y%m%d%H%M%S") +} + +# Function arguments: +# - $1 - log message to print out +log(){ + echo [$(date '+%Y-%m-%d %H:%M:%S')] $1 +} + +# Function arguments: +# - $1 - process pattern (to print it out in logs only) +# - $2 - path to program to be started +# - $3 - process options +# - $4 - log file +# - $5 - PID file to be created +start_process(){ + log "INFO Starting new [$1] process right now" + nohup $2 $3 >> $4 2>&1 & + local PROCESS_PID=$! + echo $PROCESS_PID > $5 + log "INFO New [$1] instance with pid [$PROCESS_PID] has been started, process options are [$3], pid file created: [$5]; STDOUT and STDERR redirected to [$4]" +} + +# Function arguments: +# - $1 - process pattern (to print it out in logs only) +# - $2 - PID +# - $3 - PID File +kill_process(){ + log "INFO killing [$1] process with pid [$2]" + kill $2 + if [ -f $3 ]; then + log "INFO Removing the pid file: [$3]" + rm $3 + fi +} + +# Function arguments: +# - $1 - file the message should be append on +# - $2 - log message itself +append_to_file(){ + echo $2 >> $1 +} + +# Exporting the functions +export -f log +export -f start_process +export -f kill_process +export -f append_to_file +export -f now \ No newline at end of file diff --git a/docker/release_distribution_scripts/utbot_server_restart.sh b/docker/release_distribution_scripts/utbot_server_restart.sh new file mode 100644 index 000000000..c42b0b8b8 --- /dev/null +++ b/docker/release_distribution_scripts/utbot_server_restart.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +# This script starts UTBot server + +if [ -z "$1" ] + then + export UTBOT_PORT=2121 +else + export UTBOT_PORT=$1 +fi + +# Identify the directory where the current script is located +export CURRENT_FOLDER="$( cd $( dirname ${BASH_SOURCE[0]} ) && pwd )" +# Get full path a script that launches UTBot +RUN_SYSTEM_SCRIPT_PATH=$CURRENT_FOLDER/utbot_run_system.sh +# Start script that launches UTBot +$RUN_SYSTEM_SCRIPT_PATH "server" $UTBOT_PORT \ No newline at end of file diff --git a/docker/unpack_and_run_utbot.sh b/docker/unpack_and_run_utbot.sh new file mode 100644 index 000000000..91ecc3da9 --- /dev/null +++ b/docker/unpack_and_run_utbot.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# +# Copyright (c) Huawei Technologies Co., Ltd. 2012-2021. All rights reserved. +# + +#This script unpacks and runs UTBot server +tar -xvf utbot_distr.tar.gz +cd utbot_distr +chmod +x *.sh +./utbot_server_restart.sh \ No newline at end of file diff --git a/docker/utbot_docker_dev.sh b/docker/utbot_docker_dev.sh index c06261267..583db9d33 100644 --- a/docker/utbot_docker_dev.sh +++ b/docker/utbot_docker_dev.sh @@ -11,7 +11,7 @@ MOUNT_NAME=$USER-utbot MOUNT_LOCAL_NAME=$MOUNT_NAME-local-mnt read -e -p "Enter base image tag: " IMAGE_TAG -IMAGE="docker pull ghcr.io/unittestbot/utbotcpp/base_env:$IMAGE_TAG" +IMAGE="ghcr.io/unittestbot/utbotcpp/base_env:$IMAGE_TAG" echo "Pulling docker image '$IMAGE'" if ! docker pull $IMAGE > /dev/null; then @@ -49,7 +49,6 @@ docker run -d --restart=unless-stopped \ --name=$CONTAINER_NAME \ -p $UTBOT_SSH_PORT:2020 \ -p $UTBOT_SERVER_PORT:2121 \ - -v $MOUNT_NAME:/home/utbot \ -v $MOUNT_LOCAL_NAME:/home/utbot/mnt \ -v /etc/localtime:/etc/localtime:ro \ $IMAGE > /dev/null diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 4841e7195..e9ff4acc3 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -43,20 +43,21 @@ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_COVERAGE_LINK_FLAGS} get_filename_component(UTBOT_DIR ${PROJECT_SOURCE_DIR} DIRECTORY) -set(LOGGER_DIR "${UTBOT_DIR}/loguru") +set(SUBMODULES_DIR "${UTBOT_DIR}/submodules") +set(LOGGER_DIR "${SUBMODULES_DIR}/loguru") set(LOGGER_AUX_DIR "${PROJECT_SOURCE_DIR}/logger-aux") add_library(loguru ${LOGGER_DIR}/loguru.cpp ${LOGGER_AUX_DIR}/loguru_debug.cpp) target_include_directories(loguru PUBLIC ${LOGGER_DIR}) include_directories(${LOGGER_AUX_DIR}) -set(JSON_PATH "${UTBOT_DIR}/json") +set(JSON_PATH "${SUBMODULES_DIR}/json") include_directories(${JSON_PATH}/src) -set(TSL_PATH "${UTBOT_DIR}/ordered-map") +set(TSL_PATH "${SUBMODULES_DIR}/ordered-map") message("tsl include from here: ${TSL_PATH}/include") include_directories(${TSL_PATH}/include) -set(PARALLEL_HASHMAP_PATH "${UTBOT_DIR}/parallel-hashmap") +set(PARALLEL_HASHMAP_PATH "${SUBMODULES_DIR}/parallel-hashmap") include_directories(${PARALLEL_HASHMAP_PATH}) set(KLEE_PATH $ENV{UTBOT_ALL}/klee/) diff --git a/server/build.sh b/server/build.sh index d0e815908..a81074b07 100755 --- a/server/build.sh +++ b/server/build.sh @@ -1,7 +1,5 @@ mkdir -p build cd build $UTBOT_CMAKE_BINARY -G "Ninja" -DCMAKE_INSTALL_PREFIX=$UTBOT_ALL/server-install .. -$UTBOT_CMAKE_BINARY --build . --parallel `nproc` -$UTBOT_CMAKE_BINARY --install . - -./UTBot_UnitTests info \ No newline at end of file +$UTBOT_CMAKE_BINARY --build . +$UTBOT_CMAKE_BINARY --install . \ No newline at end of file diff --git a/Bear b/submodules/Bear similarity index 100% rename from Bear rename to submodules/Bear diff --git a/json b/submodules/json similarity index 100% rename from json rename to submodules/json diff --git a/klee b/submodules/klee similarity index 100% rename from klee rename to submodules/klee diff --git a/loguru b/submodules/loguru similarity index 100% rename from loguru rename to submodules/loguru diff --git a/ordered-map b/submodules/ordered-map similarity index 100% rename from ordered-map rename to submodules/ordered-map diff --git a/parallel-hashmap b/submodules/parallel-hashmap similarity index 100% rename from parallel-hashmap rename to submodules/parallel-hashmap diff --git a/vscode-plugin/build.sh b/vscode-plugin/build.sh index 9481ab180..9d890b98f 100644 --- a/vscode-plugin/build.sh +++ b/vscode-plugin/build.sh @@ -6,8 +6,7 @@ set -e set -o pipefail utbot_dir="$(dirname $PWD)" -cp -r $UTBOT_ALL/node_modules . -npm install -cd $utbot_dir/vscode-plugin \ - && npm rebuild grpc --runtime=electron --target=7.3.0 \ - && ./protoc.sh $utbot_dir/server/proto $utbot_dir/vscode-plugin/src/proto-ts +npm install --unsafe +npm rebuild grpc --runtime=electron --target=7.3.0 +chmod +x protoc.sh && ./protoc.sh $utbot_dir/server/proto $utbot_dir/vscode-plugin/src/proto-ts +npm run compile \ No newline at end of file