From b7de7797537c3ac2207092b158a894207d9fa6cd Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Thu, 3 Sep 2020 15:44:22 -0400 Subject: [PATCH 1/9] testeroni --- CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3af4c8cf24..113e2380c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,12 +15,8 @@ set (search_paths $ENV{PATH_TO_LLVM}/share/llvm/cmake/ ) -find_package(LLVM REQUIRED CONFIG - PATHS ${search_paths} - NO_DEFAULT_PATH) -find_package(Clang REQUIRED CONFIG - PATHS ${search_paths} - NO_DEFAULT_PATH) +find_package(LLVM REQUIRED) +find_package(Clang REQUIRED) include_directories(${LLVM_INCLUDE_DIRS}) From 04e9af13fb41d944a0865fa86a05bcecef80f643 Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Thu, 3 Sep 2020 16:06:08 -0400 Subject: [PATCH 2/9] testeroni and cheese --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 113e2380c1..1091eba6ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,8 +15,12 @@ set (search_paths $ENV{PATH_TO_LLVM}/share/llvm/cmake/ ) -find_package(LLVM REQUIRED) -find_package(Clang REQUIRED) +find_package(Clang REQUIRED CONFIG + PATHS ${search_paths}) + +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") +message(STATUS "Using ClangConfig.cmake in: ${CLANG_CMAKE_DIR}") include_directories(${LLVM_INCLUDE_DIRS}) From 40077010aa13582aaecb7e8a93752f386af07bda Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Wed, 9 Sep 2020 13:35:10 -0400 Subject: [PATCH 3/9] generate compile_commands by default, make PATH_TO_LLVM a cmake var --- CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1091eba6ad..a71a338d23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,17 +2,18 @@ project(LSIFClang) cmake_minimum_required(VERSION 3.16) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/) set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) set (search_paths - $ENV{PATH_TO_LLVM} - $ENV{PATH_TO_LLVM}/lib/cmake - $ENV{PATH_TO_LLVM}/lib/cmake/llvm - $ENV{PATH_TO_LLVM}/lib/cmake/clang - $ENV{PATH_TO_LLVM}/share/clang/cmake/ - $ENV{PATH_TO_LLVM}/share/llvm/cmake/ + ${PATH_TO_LLVM} + ${PATH_TO_LLVM}/lib/cmake + ${PATH_TO_LLVM}/lib/cmake/llvm + ${PATH_TO_LLVM}/lib/cmake/clang + ${PATH_TO_LLVM}/share/clang/cmake/ + ${PATH_TO_LLVM}/share/llvm/cmake/ ) find_package(Clang REQUIRED CONFIG From a99bd6157cc1564888f7ae9e7eebb832f242ed23 Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Wed, 9 Sep 2020 13:35:25 -0400 Subject: [PATCH 4/9] update README --- README.md | 118 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 0930dc6eba..9aec66c557 100644 --- a/README.md +++ b/README.md @@ -4,72 +4,130 @@ This project is a fork of [clangd](https://clangd.llvm.org/) with patches to add This project has only been tested extensively on C++ projects, but C and Objective C projects should both be supported as well following the same instructions. -# Installation +# Alternatives for C++ Projects + +If you can't get `lsif-clang` working with your project, first file an issue! We want this to work everywhere. +But the C++ ecosystem is fragmented, and it's possible that your project simply won't play nice with the `clang` toolchain. +[lsif-cpp](https://github.com/sourcegraph/lsif-cpp) is also available, which acts as a plugin for arbitrary C++ compilers and might therefore be compatible. +But it has several major defects compared to `lsif-clang` (it is much slower and does not provide hovers), and is not the recommended option. + +# Usage + +There are 4 steps, and instructions for each can vary by platform and build system. + +1. Install dependencies +1. Build lsif-clang +1. Generate a compilation database. +1. Run lsif-clang. + +## Quick example +Here's how you would build an index of the lsif-clang tool on Ubuntu 20.04. + +```sh +apt install llvm-10 clang-10 libclang-10-dev cmake +git clone https://github.com/sourcegraph/lsif-clang && cd lsif-clang +cmake -B build +make -C build -j8 +ln -s $(pwd)/build/compile_commands.json +./build/bin/lsif-clang --project-root=$(pwd) --executor=all-TUs compile_commands.json > dump.lsif +``` + +The following sections provide detailed explanations of each step and variations on the commands for different platforms and build systems. -## Dependencies +## Install dependencies -This project depends on LLVM and Clang. The code builds against LLVM and Clang version 10, but can index a wide variety of code. Please file an issue if you need to build against a different version of LLVM or Clang and we can start adding some version pragmas! You can try finding the location of your llvm installation by running `readlink -f $(which clang)`. On my computer, this returns `/usr/lib/llvm-10/bin/clang`, so the installation path is `/usr/lib/llvm-10`. +This project depends on LLVM and Clang. lsif-clang itself should be built against LLVM and Clang version 10, and can index any code Clang 10 can compile. Work is ongoing to compile against other versions of LLVM. Here are instructions to get the dependencies on a few platforms: -## Building -The project builds with CMake, so if you know what you're doing you can configure it yourself. For sensible defaults: +### Ubuntu 20.04 ```sh -PATH_TO_LLVM= ./config.sh build +apt install llvm-10 clang-10 libclang-10-dev cmake +``` + +### MacOS + +```sh +brew install llvm cmake +``` + +## Build lsif-clang +Here is a minimal example, known extra steps for specific platforms follow: + +```sh +cmake -B build cd build make -j8 -sudo make install ``` -`PATH_TO_LLVM` should point to the llvm installation path from the previous step. The `8` in `make -j8` should be the number of threads you wish to allocate to the build (it's fairly small so it shouldn't matter much, but `make` is single threaded by default). +### MacOS +Add the following extra argument to the `cmake` step: +```sh +cmake -B build -DPATH_TO_LLVM=/usr/local/opt/lib +``` -## Give it a whirl! +## Generate a compilation database -Assuming you followed the steps above, do the following from this project's root directory: +`lsif-clang` itself is configured to do this automatically, so to test the that the tool built properly you can simply sym-link it to the project root from the build directory and skip to [running lsif-clang](). +From the project root: ```sh ln -s $(pwd)/build/compile_commands.json ./ -lsif-clang --project-root=$(pwd) --executor=all-TUs compile_commands.json > dump.lsif ``` -Inspect the file when it's done, you should see lots of glorious JSON! +Instructions for generating this compilation database for various build systems follows: -# Usage +### CMake -## Compilation Database +If a project builds with CMake, you can ensure that a compilation database is generated in the build directory with the following flag: +```sh +cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +``` -The tool depends on having a [JSON compilation database](https://clang.llvm.org/docs/JSONCompilationDatabase.html) available, which is generated differently depending on what build system is in use. Please get in touch if you're having trouble generating a compilation database, we'd be happy to help troubleshoot! +For some projects, we've noticed that CMake generates a faulty database, but that the actual build step can output more sensible values. The Ninja build tool provides a convenient mechanism for this. Assuming Ninja is installed: -### CMake - -Add `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to your `cmake` invocation. The database will get generated in your build directory, so you should symlink it to the source root. +```sh +cmake -G Ninja +cd +ninja -t compdb > compile_commands.json +``` -### Make and others +This database will also contain irrelevant entries which will make lsif-clang output quite noisy but still functional. To filter the irrelevant entries, inspect the `compile_commands.json` and find an entry for an actual c++ compile command. Here's an example for me: +```json + { + "directory": "/home/arrow/sourcegraph/lsif-clang/build2", + "command": "/usr/bin/c++ -I../ -I/usr/lib/llvm-10/include -std=gnu++17 -o CMakeFiles/clangDaemonFork.dir/AST.cpp.o -c /home/arrow/sourcegraph/lsif-clang/AST.cpp", + "file": "/home/arrow/sourcegraph/lsif-clang/AST.cpp" +} +``` -Install the [Bear](https://github.com/rizsotto/Bear) tool and run `bear make`, or `bear `. This tool is build system agnostic so it's a good fallback option. +I can then use the "command" value of `/usr/bin/c++` in the following jq snippet: +```sh +ninja -t compdb | jq '[ .[] | select(.command | startswith("/usr/bin/c++")) ] > compile_commands.json' +``` ### Bazel Use the [bazel-compilation-database](https://github.com/grailbio/bazel-compilation-database) tool. -## Indexing +### Make and others + +Install the [Bear](https://github.com/rizsotto/Bear) tool and run `bear make`, or `bear `. This tool is build system agnostic so it's a good fallback option. + +## Run lsif-clang -Once you have `compile_commands.json` at the root of your source tree, you can invoke `lsif-clang` like so: +Once you have a `compile_commands.json` in the root of your project's source, you can use the following command to index the entire project: ```sh lsif-clang --project-root=$(pwd) --executor=all-TUs compile_commands.json > dump.lsif ``` -This will index the entire project. To index only some files, run: +To index individual files, use: ```sh lsif-clang --project-root=$(pwd) file1.cpp file2.cpp ... > dump.lsif ``` -Note that this will still include lots of data about other files to properly supply hovers and such. - -### Platform-specific instructions - -#### On MacOS +### MacOS The indexer may fail to find system header files on MacOS (and possibly other systems) resulting in console error messages such as `fatal error: stdarg.h not found`. @@ -85,6 +143,6 @@ $ lsif-clang \ compile_commands.json > dump.lsif ``` -# Alternatives for C++ Projects +## Test the output -If you can't get `lsif-clang` working with your project, first file an issue! We want this to work everywhere. But the C++ ecosystem is fragmented, and it's possible that your project simply won't play nice with the `clang` toolchain. [lsif-cpp](https://github.com/sourcegraph/lsif-cpp) is also available, which acts as a plugin for arbitrary C++ compilers and might therefore be compatible. But it has several major defects compared to `lsif-clang` (it is much slower and does not provide hovers), and is not the recommended option. +You can use the [lsif-validate]() tool for basic sanity checking, or [upload the index to a Sourcegraph instance]() to see the hovers, definitions, and references in action. From db9c968bdb62d10ebc30fb96a578de711e9df6da Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Wed, 9 Sep 2020 13:35:49 -0400 Subject: [PATCH 5/9] remove config script (obviated by more ergonomic CMakeLists.txt) --- config.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 config.sh diff --git a/config.sh b/config.sh deleted file mode 100755 index ec91804c1a..0000000000 --- a/config.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -cmake -B $1 -S . -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Release From 39146819c950f6e43011f2ae8ba56720ff789d43 Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Wed, 9 Sep 2020 13:40:27 -0400 Subject: [PATCH 6/9] more readme improvements --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9aec66c557..57fe80ff8c 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,12 @@ There are 4 steps, and instructions for each can vary by platform and build syst Here's how you would build an index of the lsif-clang tool on Ubuntu 20.04. ```sh -apt install llvm-10 clang-10 libclang-10-dev cmake -git clone https://github.com/sourcegraph/lsif-clang && cd lsif-clang -cmake -B build -make -C build -j8 -ln -s $(pwd)/build/compile_commands.json -./build/bin/lsif-clang --project-root=$(pwd) --executor=all-TUs compile_commands.json > dump.lsif +apt install llvm-10 clang-10 libclang-10-dev cmake `# install dependencies` +git clone https://github.com/sourcegraph/lsif-clang && cd lsif-clang `# get the code` +cmake -B build `# configure lsif-clang` +make -C build -j8 `# build lsif-clang` +ln -s $(pwd)/build/compile_commands.json ./ `# link the compilation database to the project root` +./build/bin/lsif-clang --project-root=$(pwd) --executor=all-TUs compile_commands.json > dump.lsif `# generate an index` ``` The following sections provide detailed explanations of each step and variations on the commands for different platforms and build systems. From 5be9e828f2fef2b11ead87f2fe0088d62710aa4b Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Wed, 9 Sep 2020 13:59:22 -0400 Subject: [PATCH 7/9] minor readme update --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 57fe80ff8c..365e27bb99 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ There are 4 steps, and instructions for each can vary by platform and build syst Here's how you would build an index of the lsif-clang tool on Ubuntu 20.04. ```sh -apt install llvm-10 clang-10 libclang-10-dev cmake `# install dependencies` +apt install llvm-10 clang clang-10 libclang-10-dev cmake `# install dependencies` git clone https://github.com/sourcegraph/lsif-clang && cd lsif-clang `# get the code` cmake -B build `# configure lsif-clang` make -C build -j8 `# build lsif-clang` @@ -41,7 +41,7 @@ This project depends on LLVM and Clang. lsif-clang itself should be built agains ### Ubuntu 20.04 ```sh -apt install llvm-10 clang-10 libclang-10-dev cmake +apt install llvm-10 clang clang-10 libclang-10-dev cmake ``` ### MacOS @@ -55,8 +55,7 @@ Here is a minimal example, known extra steps for specific platforms follow: ```sh cmake -B build -cd build -make -j8 +make -C build -j8 ``` ### MacOS From 76b97357a6e486f050b9c8a79fe140a464e30d9d Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Mon, 21 Sep 2020 11:10:39 -0400 Subject: [PATCH 8/9] Revert "remove config script (obviated by more ergonomic CMakeLists.txt)" This reverts commit db9c968bdb62d10ebc30fb96a578de711e9df6da. --- config.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 config.sh diff --git a/config.sh b/config.sh new file mode 100755 index 0000000000..ec91804c1a --- /dev/null +++ b/config.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cmake -B $1 -S . -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Release From 2e40f1aea4c9bc748a5ecbe9fefc07305e9e5e14 Mon Sep 17 00:00:00 2001 From: Garo Brik Date: Mon, 21 Sep 2020 11:17:04 -0400 Subject: [PATCH 9/9] make config script backwards compatible --- CMakeLists.txt | 20 ++++++++++++-------- config.sh | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a71a338d23..2b235270c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,14 +7,18 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/) set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -set (search_paths - ${PATH_TO_LLVM} - ${PATH_TO_LLVM}/lib/cmake - ${PATH_TO_LLVM}/lib/cmake/llvm - ${PATH_TO_LLVM}/lib/cmake/clang - ${PATH_TO_LLVM}/share/clang/cmake/ - ${PATH_TO_LLVM}/share/llvm/cmake/ -) +if (PATH_TO_LLVM) + set (search_paths + ${PATH_TO_LLVM} + ${PATH_TO_LLVM}/lib/cmake + ${PATH_TO_LLVM}/lib/cmake/llvm + ${PATH_TO_LLVM}/lib/cmake/clang + ${PATH_TO_LLVM}/share/clang/cmake/ + ${PATH_TO_LLVM}/share/llvm/cmake/ + ) +endif() + +message(STATUS "${search_paths}") find_package(Clang REQUIRED CONFIG PATHS ${search_paths}) diff --git a/config.sh b/config.sh index ec91804c1a..e9cfa54c77 100755 --- a/config.sh +++ b/config.sh @@ -1,3 +1,5 @@ #!/bin/sh -cmake -B $1 -S . -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Release +cmake -B build -S . \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DPATH_TO_LLVM=$PATH_TO_LLVM