diff --git a/.github/workflows/trigger-build.yml b/.github/workflows/trigger-build.yml new file mode 100644 index 0000000..b4b087c --- /dev/null +++ b/.github/workflows/trigger-build.yml @@ -0,0 +1,16 @@ +name: Trigger RocksDB build on nethermind repo + +on: + push: + branches: + - master + +jobs: + bls-building: + name: Building RocksDB + runs-on: ubuntu-latest + steps: + - name: Send dispatch event to nethermind repo + run: | + curl -v -X POST -u "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" --data '{"event_type":"rocksdb_build"}' https://api.github.com/repos/nethermindeth/nethermind/dispatches + diff --git a/.gitignore b/.gitignore index efd4bc0..70cd40d 100644 --- a/.gitignore +++ b/.gitignore @@ -179,14 +179,13 @@ $RECYCLE.BIN/ # SASS Compiler cache .sass-cache +# Rider +.idea/ + # Visual Studio 2014 CTP **/*.sln.ide .vs/ -amd64/rocksdb.dll -native/**/*.dll -native/ -native*/ *.xz *.zip *.userprefs diff --git a/RocksDbNative/RocksDbNative.csproj b/RocksDbNative/RocksDbNative.csproj index 57fb330..8409354 100644 --- a/RocksDbNative/RocksDbNative.csproj +++ b/RocksDbNative/RocksDbNative.csproj @@ -1,8 +1,7 @@  - + RocksDbNative - netstandard1.6;net40;net45 $(RocksDbVersion).$(RocksDbNativeBuild) $(Version) $(Version) @@ -18,51 +17,59 @@ Copyright 2016 Native RocksDb Binaries (Install RocksDbSharp package for managed wraper) False + netstandard2.1 + latest + true - - - - - - - + + + + + + + - - - + + + + + + + - - true - PreserveNewest - - - - true - build/net40 - - - true - build/net45 - + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + \ No newline at end of file diff --git a/RocksDbNative/build/net40/RocksDbNative.targets b/RocksDbNative/build/net40/RocksDbNative.targets deleted file mode 100644 index 07eac99..0000000 --- a/RocksDbNative/build/net40/RocksDbNative.targets +++ /dev/null @@ -1,20 +0,0 @@ - - - - - PreserveNewest - runtimes\win-x64\native\rocksdb.dll - false - - - PreserveNewest - runtimes\linux-x64\native\librocksdb.so - false - - - PreserveNewest - runtimes\osx-x64\native\librocksdb.dylib - false - - - \ No newline at end of file diff --git a/RocksDbNative/build/net45/RocksDbNative.targets b/RocksDbNative/build/net45/RocksDbNative.targets deleted file mode 100644 index 07eac99..0000000 --- a/RocksDbNative/build/net45/RocksDbNative.targets +++ /dev/null @@ -1,20 +0,0 @@ - - - - - PreserveNewest - runtimes\win-x64\native\rocksdb.dll - false - - - PreserveNewest - runtimes\linux-x64\native\librocksdb.so - false - - - PreserveNewest - runtimes\osx-x64\native\librocksdb.dylib - false - - - \ No newline at end of file diff --git a/RocksDbNative/runtimes/linux-arm/native/librocksdb.so b/RocksDbNative/runtimes/linux-arm/native/librocksdb.so new file mode 100644 index 0000000..611aa85 Binary files /dev/null and b/RocksDbNative/runtimes/linux-arm/native/librocksdb.so differ diff --git a/RocksDbNative/runtimes/linux-arm64/native/librocksdb.so b/RocksDbNative/runtimes/linux-arm64/native/librocksdb.so new file mode 100755 index 0000000..45acecf Binary files /dev/null and b/RocksDbNative/runtimes/linux-arm64/native/librocksdb.so differ diff --git a/RocksDbNative/runtimes/linux-x64/native/librocksdb.so b/RocksDbNative/runtimes/linux-x64/native/librocksdb.so new file mode 100644 index 0000000..47a689d Binary files /dev/null and b/RocksDbNative/runtimes/linux-x64/native/librocksdb.so differ diff --git a/RocksDbNative/runtimes/osx-arm64/native/librocksdb.dylib b/RocksDbNative/runtimes/osx-arm64/native/librocksdb.dylib new file mode 100755 index 0000000..2b09d3c Binary files /dev/null and b/RocksDbNative/runtimes/osx-arm64/native/librocksdb.dylib differ diff --git a/RocksDbNative/runtimes/osx-x64/native/librocksdb.dylib b/RocksDbNative/runtimes/osx-x64/native/librocksdb.dylib new file mode 100644 index 0000000..c7b426f Binary files /dev/null and b/RocksDbNative/runtimes/osx-x64/native/librocksdb.dylib differ diff --git a/RocksDbNative/runtimes/win-x64/native/rocksdb.dll b/RocksDbNative/runtimes/win-x64/native/rocksdb.dll new file mode 100644 index 0000000..d080b69 Binary files /dev/null and b/RocksDbNative/runtimes/win-x64/native/rocksdb.dll differ diff --git a/RocksDbSharp/AutoNativeImport.cs b/RocksDbSharp/AutoNativeImport.cs index 40a4a25..6d8ffab 100644 --- a/RocksDbSharp/AutoNativeImport.cs +++ b/RocksDbSharp/AutoNativeImport.cs @@ -192,7 +192,7 @@ public static string GetArchName(Architecture arch) case Architecture.X86: return "i386"; case Architecture.X64: - return "amd64"; + return "x64"; default: return arch.ToString().ToLower(); } @@ -368,6 +368,20 @@ public static T Import(INativeLibImporter importer, string libName, string ve "", }; + HashSet basePaths = new(); + void TryAddBasePath(string path) + { + if (path is not null) + { + try { basePaths.Add(path); } + catch + { + /* Ignore */ + } + } + } + + // If the RocksDbNative package is referenced, then dynamically load it here so that it can tell us where the native libraries are string nativeCodeBase = null; try @@ -379,18 +393,24 @@ public static T Import(INativeLibImporter importer, string libName, string ve var getCodeBase = getCodeBaseMethod.CreateDelegate>(); nativeCodeBase = getCodeBase(); } - catch (Exception) + catch { /* Ignore */ } + + //Some paths might throw NotSupportedException when used from single file deployment. We could test for that, but we can also just ignore it + TryAddBasePath(nativeCodeBase); + TryAddBasePath(Directory.GetCurrentDirectory()); + TryAddBasePath(Path.GetDirectoryName(UriToPath(AppContext.BaseDirectory))); + TryAddBasePath(Path.GetDirectoryName(UriToPath(Transitional.CurrentFramework.GetBaseDirectory()))); + TryAddBasePath(Path.GetDirectoryName(UriToPath(Assembly.GetEntryAssembly()?.Location))); + TryAddBasePath(Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location)); + TryAddBasePath(Path.GetDirectoryName(UriToPath(typeof(PosixImporter).GetTypeInfo().Assembly.Location))); + TryAddBasePath(Path.GetDirectoryName(typeof(PosixImporter).GetTypeInfo().Assembly.Location)); + + // for Apple ARM64 - M1 + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { - } + TryAddBasePath("/opt/homebrew/lib"); + } - var basePaths = new string[] { - nativeCodeBase, - Path.GetDirectoryName(UriToPath(Transitional.CurrentFramework.GetBaseDirectory())), - Path.GetDirectoryName(UriToPath(Assembly.GetEntryAssembly()?.CodeBase)), - Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location), - Path.GetDirectoryName(UriToPath(typeof(PosixImporter).GetTypeInfo().Assembly.CodeBase)), - Path.GetDirectoryName(typeof(PosixImporter).GetTypeInfo().Assembly.Location), - }; var search = basePaths .Where(p => p != null) .Distinct() diff --git a/RocksDbSharp/Native.Load.cs b/RocksDbSharp/Native.Load.cs index 8f2ed8c..15a5808 100644 --- a/RocksDbSharp/Native.Load.cs +++ b/RocksDbSharp/Native.Load.cs @@ -15,7 +15,7 @@ static Native() { if (RuntimeInformation.ProcessArchitecture == Architecture.X86 && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) throw new RocksDbSharpException("Rocksdb on windows is not supported for 32 bit applications"); - Instance = NativeImport.Auto.Import("rocksdb", "6.2.2", true); + Instance = NativeImport.Auto.Import("rocksdb", "6.4.6", true); } public Native() diff --git a/RocksDbSharp/Native.Marshaled.cs b/RocksDbSharp/Native.Marshaled.cs index 503a9d1..de7e72d 100644 --- a/RocksDbSharp/Native.Marshaled.cs +++ b/RocksDbSharp/Native.Marshaled.cs @@ -1,10 +1,11 @@ -/* +/* The functions in this file provide some wrappers around the lowest level C API to aid in marshalling. This is kept separate so that the lowest level imports can be kept as close as possible to c.h from rocksdb. See Native.Raw.cs for more information. */ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using Transitional; @@ -155,7 +156,37 @@ public byte[] rocksdb_get( rocksdb_free(resultPtr); return result; } - + + public unsafe Span rocksdb_get_span( + IntPtr db, + IntPtr read_options, + byte[] key, + long keyLength, + out IntPtr errptr, + ColumnFamilyHandle cf = null) + { + UIntPtr skLength = (UIntPtr)keyLength; + var resultPtr = cf == null + ? rocksdb_get(db, read_options, key, skLength, out UIntPtr valueLength, out errptr) + : rocksdb_get_cf(db, read_options, cf.Handle, key, skLength, out valueLength, out errptr); + if (errptr != IntPtr.Zero) + return null; + if (resultPtr == IntPtr.Zero) + return null; + Span span = new Span((void*)resultPtr, (int)valueLength); + //MemoryMarshal.GetReference(span); + + + return span; + } + + public unsafe void rocksdb_release_span(in Span span) + { + ref byte ptr = ref MemoryMarshal.GetReference(span); + IntPtr intPtr = new IntPtr(Unsafe.AsPointer(ref ptr)); + rocksdb_free(intPtr); + } + /// /// Executes a multi_get with automatic marshalling /// diff --git a/RocksDbSharp/Native.Wrap.cs b/RocksDbSharp/Native.Wrap.cs index 4643f87..7bd150b 100644 --- a/RocksDbSharp/Native.Wrap.cs +++ b/RocksDbSharp/Native.Wrap.cs @@ -44,6 +44,26 @@ public void rocksdb_put( if (errptr != IntPtr.Zero) throw new RocksDbException(errptr); } + + public unsafe void rocksdb_put( + IntPtr db, + IntPtr writeOptions, + byte* key, + long keyLength, + byte* value, + long valueLength, + ColumnFamilyHandle cf) + { + IntPtr errptr; + UIntPtr sklength = (UIntPtr)keyLength; + UIntPtr svlength = (UIntPtr)valueLength; + if (cf == null) + rocksdb_put(db, writeOptions, key, sklength, value, svlength, out errptr); + else + rocksdb_put_cf(db, writeOptions, cf.Handle, key, sklength, value, svlength, out errptr); + if (errptr != IntPtr.Zero) + throw new RocksDbException(errptr); + } public string rocksdb_get( @@ -89,6 +109,19 @@ public byte[] rocksdb_get( throw new RocksDbException(errptr); return result; } + + public Span rocksdb_get_span( + IntPtr db, + IntPtr read_options, + byte[] key, + long keyLength = 0, + ColumnFamilyHandle cf = null) + { + var result = rocksdb_get_span(db, read_options, key, keyLength == 0 ? key.Length : keyLength, out IntPtr errptr, cf); + if (errptr != IntPtr.Zero) + throw new RocksDbException(errptr); + return result; + } public System.Collections.Generic.KeyValuePair[] rocksdb_multi_get( IntPtr db, diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index 770898b..ec2baac 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -119,12 +119,22 @@ public byte[] Get(byte[] key, ColumnFamilyHandle cf = null, ReadOptions readOpti { return Get(key, key.GetLongLength(0), cf, readOptions); } - + public byte[] Get(byte[] key, long keyLength, ColumnFamilyHandle cf = null, ReadOptions readOptions = null) { return Native.Instance.rocksdb_get(Handle, (readOptions ?? DefaultReadOptions).Handle, key, keyLength, cf); } + public Span GetSpan(byte[] key, ColumnFamilyHandle cf = null, ReadOptions readOptions = null) + { + return GetSpan(key, key.GetLongLength(0), cf, readOptions); + } + + public Span GetSpan(byte[] key, long keyLength, ColumnFamilyHandle cf = null, ReadOptions readOptions = null) + { + return Native.Instance.rocksdb_get_span(Handle, (readOptions ?? DefaultReadOptions).Handle, key, keyLength, cf); + } + /// /// Reads the contents of the database value associated with , if present, into the supplied /// at up to bytes, returning the @@ -215,11 +225,24 @@ public void Put(byte[] key, byte[] value, ColumnFamilyHandle cf = null, WriteOpt { Put(key, key.GetLongLength(0), value, value.GetLongLength(0), cf, writeOptions); } + + public void Put(Span key, Span value, ColumnFamilyHandle cf = null, WriteOptions writeOptions = null) + { + Put(key, key.Length, value, value.Length, cf, writeOptions); + } public void Put(byte[] key, long keyLength, byte[] value, long valueLength, ColumnFamilyHandle cf = null, WriteOptions writeOptions = null) { Native.Instance.rocksdb_put(Handle, (writeOptions ?? DefaultWriteOptions).Handle, key, keyLength, value, valueLength, cf); } + + public unsafe void Put(Span key, long keyLength, Span value, long valueLength, ColumnFamilyHandle cf = null, WriteOptions writeOptions = null) + { + fixed (byte* keyPtr = &MemoryMarshal.GetReference(key), valuePtr = &MemoryMarshal.GetReference(value)) + { + Native.Instance.rocksdb_put(Handle, (writeOptions ?? DefaultWriteOptions).Handle, keyPtr, keyLength, valuePtr, valueLength, cf); + } + } public Iterator NewIterator(ColumnFamilyHandle cf = null, ReadOptions readOptions = null) { @@ -306,5 +329,10 @@ public void CompactRange(string start, string limit, ColumnFamilyHandle cf = nul encoding = Encoding.UTF8; CompactRange(start == null ? null : encoding.GetBytes(start), limit == null ? null : encoding.GetBytes(limit), cf); } + + public void DangerousReleaseMemory(in Span span) + { + Native.Instance.rocksdb_release_span(in span); + } } } diff --git a/RocksDbSharp/RocksDbSharp.csproj b/RocksDbSharp/RocksDbSharp.csproj index 9f03eff..7dcb630 100644 --- a/RocksDbSharp/RocksDbSharp.csproj +++ b/RocksDbSharp/RocksDbSharp.csproj @@ -1,8 +1,7 @@  - + RocksDbSharp - netstandard1.6;net40;net45 True $(RocksDbVersion).$(RocksDbSharpBuild) $(Version) @@ -19,29 +18,29 @@ Copyright 2016 .Net Bindings for RocksDb. See the Project Site for more information. (Note: Also install RocksDbNative package to have native binaries included with build) False + net6.0 + latest + true - - - TRACE;DEBUG + TRACE;DEBUG;ROCKSDB_CACHE - - TRACE + TRACE;ROCKSDB_CACHE - - - - - - \ No newline at end of file + + + + + + diff --git a/RocksDbSharp/Transitional.cs b/RocksDbSharp/Transitional.cs index 936489a..17323f0 100644 --- a/RocksDbSharp/Transitional.cs +++ b/RocksDbSharp/Transitional.cs @@ -139,46 +139,46 @@ public unsafe static string GetString(this Encoding encoding, byte* bytes, int c } -#if !NETSTANDARD1_6 -namespace System.Runtime.InteropServices -{ - public static class OSPlatform - { - public static string Linux { get; } = "Linux"; - public static string OSX { get; } = "OSX"; - public static string Windows { get; } = "Windows"; - } - - public enum Architecture - { - X86 = 0, - X64 = 1, - Arm = 2, - Arm64 = 3 - } - - internal static class RuntimeInformation - { - internal static bool IsOSPlatform(string osplatform) - { - switch ((int)Environment.OSVersion.Platform) - { - case (int)PlatformID.Win32Windows: // Win9x supported? - case (int)PlatformID.Win32S: // Win16 NTVDM on Win x86? - case (int)PlatformID.Win32NT: // Windows NT - case (int)PlatformID.WinCE: - return osplatform == OSPlatform.Windows; - case (int)PlatformID.Unix: - return osplatform == OSPlatform.Linux; - case (int)PlatformID.MacOSX: - case 128: // Mono Mac - return osplatform == OSPlatform.OSX; - default: - return false; - } - } - - internal static Architecture ProcessArchitecture => Environment.Is64BitProcess? Architecture.X64 : Architecture.X86; - } -} -#endif \ No newline at end of file +//#if !NETSTANDARD1_6 +//namespace System.Runtime.InteropServices +//{ +// public static class OSPlatform +// { +// public static string Linux { get; } = "Linux"; +// public static string OSX { get; } = "OSX"; +// public static string Windows { get; } = "Windows"; +// } +// +// public enum Architecture +// { +// X86 = 0, +// X64 = 1, +// Arm = 2, +// Arm64 = 3 +// } +// +// internal static class RuntimeInformation +// { +// internal static bool IsOSPlatform(string osplatform) +// { +// switch ((int)Environment.OSVersion.Platform) +// { +// case (int)PlatformID.Win32Windows: // Win9x supported? +// case (int)PlatformID.Win32S: // Win16 NTVDM on Win x86? +// case (int)PlatformID.Win32NT: // Windows NT +// case (int)PlatformID.WinCE: +// return osplatform == OSPlatform.Windows; +// case (int)PlatformID.Unix: +// return osplatform == OSPlatform.Linux; +// case (int)PlatformID.MacOSX: +// case 128: // Mono Mac +// return osplatform == OSPlatform.OSX; +// default: +// return false; +// } +// } +// +// internal static Architecture ProcessArchitecture => Environment.Is64BitProcess? Architecture.X64 : Architecture.X86; +// } +//} +//#endif \ No newline at end of file diff --git a/examples/ColumnFamilyExample/ColumnFamilyExample.csproj b/examples/ColumnFamilyExample/ColumnFamilyExample.csproj index 19db3e5..d912ea9 100644 --- a/examples/ColumnFamilyExample/ColumnFamilyExample.csproj +++ b/examples/ColumnFamilyExample/ColumnFamilyExample.csproj @@ -1,8 +1,9 @@  - net45 + net6.0 Exe x64 + latest diff --git a/examples/PrefixExample/PrefixExample.csproj b/examples/PrefixExample/PrefixExample.csproj index 8a58534..bd1c639 100644 --- a/examples/PrefixExample/PrefixExample.csproj +++ b/examples/PrefixExample/PrefixExample.csproj @@ -1,9 +1,10 @@  - net45 + net6.0 Exe x64 1.0.0 + latest diff --git a/examples/SimpleExampleHighLevel/SimpleExampleHighLevel.csproj b/examples/SimpleExampleHighLevel/SimpleExampleHighLevel.csproj index 19db3e5..d912ea9 100644 --- a/examples/SimpleExampleHighLevel/SimpleExampleHighLevel.csproj +++ b/examples/SimpleExampleHighLevel/SimpleExampleHighLevel.csproj @@ -1,8 +1,9 @@  - net45 + net6.0 Exe x64 + latest diff --git a/examples/SimpleExampleLowLevel/SimpleExampleLowLevel.csproj b/examples/SimpleExampleLowLevel/SimpleExampleLowLevel.csproj index 19db3e5..d912ea9 100644 --- a/examples/SimpleExampleLowLevel/SimpleExampleLowLevel.csproj +++ b/examples/SimpleExampleLowLevel/SimpleExampleLowLevel.csproj @@ -1,8 +1,9 @@  - net45 + net6.0 Exe x64 + latest diff --git a/native-build/build-rocksdb.sh b/native-build/build-rocksdb.sh new file mode 100755 index 0000000..eda6155 --- /dev/null +++ b/native-build/build-rocksdb.sh @@ -0,0 +1,293 @@ +#!/bin/bash +# WINDOWS: +# If you are in Windows, this is designed to be run from git bash +# You therefore should install git bash, Visual Studio 2015, and cmake +# Your best bet in Windows is to open a Developer Command Prompt and then run bash from there. +# MAC: +# You will need snappy (must build: homebrew version is not universal) +# brew install automake +# brew install libtool +# git clone git@github.com:google/snappy.git +# cd snappy +# ./autogen.sh +# ./configure --prefix=/usr/local CFLAGS="-arch i386 -arch x86_64" CXXFLAGS="-arch i386 -arch x86_64" LDFLAGS="-arch i386 -arch x86_64" --disable-dependency-tracking +# make +# sudo make install +# +# Instructions for upgrading rocksdb version +# 1. Fetch the desired version locally with something like: +# cd native-build/rocksdb +# git fetch https://github.com/facebook/rocksdb.git v4.13 +# git checkout FETCH_HEAD +# git push -f warrenfalk HEAD:rocksdb_sharp +# 2. Get the hash of the commit for the version and replace below +# 3. Also see instructions for modifying Native.Raw.cs with updates to c.h since current revisions +# 4. Push the desired version to the rocksdb_sharp branch at https://github.com/warrenfalk/rocksdb +# 5. Search through code for old hash and old version number and replace +# 6. Run this script to build (see README.md for more info) + +#ROCKSDBVERSION=6e0597951 +ROCKSDBVERSION=tags/v5.15.10 +ROCKSDBVNUM=5.15.10 +ROCKSDBSHARPVNUM=5.15.10.10 +SNAPPYVERSION=37aafc9e + +ROCKSDBREMOTE=https://github.com/facebook/rocksdb +SNAPPYREMOTE=https://github.com/warrenfalk/snappy + +CONCURRENCY=8 + +fail() { + echo "FAILURE" + >&2 echo -e "\033[1;31m$1\033[0m" +} + +warn() { + >&2 echo -e "\033[1;33m$1\033[0m" +} + +run_rocksdb_test() { + NAME=$1 + echo "Running test \"${NAME}\":" + cmd //c "build\\Debug\\${NAME}.exe" || fail "Test failed" +} + +checkout() { + NAME="$1" + REMOTE="$2" + VERSION="$3" + FETCHREF="$4" + test -d .git || git init + test -d .git || fail "unable to initialize $NAME repository" + git fetch "$REMOTE" "${FETCHREF}" || fail "Unable to fetch latest ${FETCHREF} from {$REMOTE}" + git checkout "$VERSION" || fail "Unable to checkout $NAME version ${VERSION}" +} + +update_vcxproj(){ + echo "Patching vcxproj for static vc runtime" + /bin/find . -type f -name '*.vcxproj' -exec sed -i 's/MultiThreadedDLL/MultiThreaded/g; s/MultiThreadedDebugDLL/MultiThreadedDebug/g' '{}' ';' +} + +BASEDIR=$(dirname "$0") +OSINFO=$(uname) + +if [[ $OSINFO == *"MSYS"* || $OSINFO == *"MINGW"* ]]; then + echo "Detected Windows (MSYS)..." + # Make sure git is installed + hash git 2> /dev/null || { fail "Build requires git"; } + test -z "$WindowsSdkDir" && fail "This must be run from a build environment such as the Developer Command Prompt" + + BASEDIRWIN=$(cd "${BASEDIR}" && pwd -W) + + mkdir -p snappy || fail "unable to create snappy directory" + (cd snappy && { + checkout "snappy" "$SNAPPYREMOTE" "$SNAPPYVERSION" "cmake" + mkdir -p build + (cd build && { + cmake -G "Visual Studio 14 2015 Win64" .. || fail "Running cmake on snappy failed" + update_vcxproj || warn "unable to patch snappy for static vc runtime" + }) || fail "cmake build generation failed" + + test -z "$RUNTESTS" || { + cmd //c "msbuild build/snappy.sln /p:Configuration=Debug /m:$CONCURRENCY" || fail "Build of snappy (debug config) failed" + } + cmd //c "msbuild build/snappy.sln /p:Configuration=Release /m:$CONCURRENCY" || fail "Build of snappy failed" + }) || fail "Snappy build failed" + + + #mkdir -p rocksdb || fail "unable to create rocksdb directory" + rm -rf rocksdb + git clone "$ROCKSDBREMOTE" || fail "unable to clone rocksdb" + (cd rocksdb && { + + git checkout "$ROCKSDBVERSION" + #checkout "rocksdb" "$ROCKSDBREMOTE" "$ROCKSDBVERSION" "rocksdb_sharp" + + git checkout -- thirdparty.inc + patch -N < ../rocksdb.thirdparty.inc.patch || warn "Patching of thirdparty.inc failed" + rm -f thirdparty.inc.rej thirdparty.inc.orig + + # The following was necessary to get it to stop trying to build the tools + # which doesn't work because of relative paths to GFLAGS library + git checkout -- CMakeLists.txt + patch -N < ../rocksdb.nobuildtools.patch || warn "Patching of CMakeLists.txt failed" + rm -f CMakeLists.txt.rej CMakeLists.txt.orig + + sed -i 's/\/MD/\/MT/g' CMakeLists.txt + + mkdir -p build + (cd build && { + cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DGFLAGS=0 -DSNAPPY=1 -DPORTABLE=1 -DWITH_AVX2=0 .. || fail "Running cmake failed" + update_vcxproj || warn "failed to patch vcxproj files for static vc runtime" + }) || fail "cmake build generation failed" + + export TEST_TMPDIR=$(cmd //c "echo %TMP%") + + # TODO: build debug version first and run tests + test -z "$RUNTESTS" || { + cmd //c "msbuild build/rocksdb.sln /p:Configuration=Debug /m:$CONCURRENCY" || fail "Rocksdb debug build failed" + run_rocksdb_test db_test + run_rocksdb_test db_iter_test + run_rocksdb_test db_log_iter_test + run_rocksdb_test db_compaction_filter_test + run_rocksdb_test db_compaction_test + run_rocksdb_test db_dynamic_level_test + run_rocksdb_test db_inplace_update_test + run_rocksdb_test db_tailing_iter_test + run_rocksdb_test db_universal_compaction_test + run_rocksdb_test db_wal_test + run_rocksdb_test db_table_properties_test + run_rocksdb_test block_hash_index_test + run_rocksdb_test autovector_test + run_rocksdb_test column_family_test + run_rocksdb_test table_properties_collector_test + run_rocksdb_test arena_test + run_rocksdb_test auto_roll_logger_test + run_rocksdb_test block_test + run_rocksdb_test bloom_test + run_rocksdb_test dynamic_bloom_test + run_rocksdb_test c_test + run_rocksdb_test cache_test + run_rocksdb_test checkpoint_test + run_rocksdb_test coding_test + run_rocksdb_test corruption_test + run_rocksdb_test crc32c_test + run_rocksdb_test slice_transform_test + run_rocksdb_test dbformat_test + run_rocksdb_test env_test + run_rocksdb_test fault_injection_test + run_rocksdb_test filelock_test + run_rocksdb_test filename_test + run_rocksdb_test file_reader_writer_test + run_rocksdb_test block_based_filter_block_test + run_rocksdb_test full_filter_block_test + run_rocksdb_test histogram_test + run_rocksdb_test inlineskiplist_test + run_rocksdb_test log_test + run_rocksdb_test manual_compaction_test + run_rocksdb_test memenv_test + run_rocksdb_test mock_env_test + run_rocksdb_test memtable_list_test + run_rocksdb_test merge_helper_test + run_rocksdb_test memory_test + run_rocksdb_test merge_test + run_rocksdb_test merger_test + run_rocksdb_test options_file_test + run_rocksdb_test redis_lists_test + run_rocksdb_test reduce_levels_test + run_rocksdb_test plain_table_db_test + run_rocksdb_test comparator_db_test + run_rocksdb_test prefix_test + run_rocksdb_test skiplist_test + run_rocksdb_test stringappend_test + run_rocksdb_test ttl_test + run_rocksdb_test backupable_db_test + run_rocksdb_test document_db_test + run_rocksdb_test json_document_test + run_rocksdb_test spatial_db_test + run_rocksdb_test version_edit_test + run_rocksdb_test version_set_test + run_rocksdb_test compaction_picker_test + run_rocksdb_test version_builder_test + run_rocksdb_test file_indexer_test + run_rocksdb_test write_batch_test + run_rocksdb_test write_batch_with_index_test + run_rocksdb_test write_controller_test + run_rocksdb_test deletefile_test + run_rocksdb_test table_test + run_rocksdb_test thread_local_test + run_rocksdb_test geodb_test + run_rocksdb_test rate_limiter_test + run_rocksdb_test delete_scheduler_test + run_rocksdb_test options_test + run_rocksdb_test options_util_test + run_rocksdb_test event_logger_test + run_rocksdb_test cuckoo_table_builder_test + run_rocksdb_test cuckoo_table_reader_test + run_rocksdb_test cuckoo_table_db_test + run_rocksdb_test flush_job_test + run_rocksdb_test wal_manager_test + run_rocksdb_test listener_test + run_rocksdb_test compaction_iterator_test + run_rocksdb_test compaction_job_test + run_rocksdb_test thread_list_test + run_rocksdb_test sst_dump_test + run_rocksdb_test compact_files_test + run_rocksdb_test perf_context_test + run_rocksdb_test optimistic_transaction_test + run_rocksdb_test write_callback_test + run_rocksdb_test heap_test + run_rocksdb_test compact_on_deletion_collector_test + run_rocksdb_test compaction_job_stats_test + run_rocksdb_test transaction_test + run_rocksdb_test ldb_cmd_test + } + cmd //c "msbuild build/rocksdb.sln /p:Configuration=Release /m:$CONCURRENCY" || fail "Rocksdb release build failed" + git checkout -- thirdparty.inc + mkdir -p ../../native/amd64 && cp -v ./build/Release/rocksdb_shared.dll ../../native/amd64/rocksdb.dll + mkdir -p ../../native-${ROCKSDBVERSION}/amd64 && cp -v ./build/Release/rocksdb.dll ../../native-${ROCKSDBVERSION}/amd64/rocksdb.dll + }) || fail "rocksdb build failed" +elif [[ $OSDETECT == *"Darwin"* ]]; then + fail "Mac OSX build is not yet operational" +else + echo "Assuming a posix-like environment" + if [ "$(uname)" == "Darwin" ]; then + echo "Mac (Darwin) detected" + CFLAGS=-I/usr/local/include + LDFLAGS="-L/usr/local/lib" + LIBEXT=.dylib + else + CFLAGS=-static-libstdc++ + LIBEXT=.so + fi + # Linux Dependencies + #sudo apt-get install gcc-5-multilib g++-5-multilib + #sudo apt-get install libsnappy-dev:i386 libbz2-dev:i386 libsnappy-dev libbz2-dev + + # Mac Dependencies + #brew install snappy --universal + # Don't have universal version of lz4 through brew, have to build manually + # git clone git@github.com:Cyan4973/lz4.git + # make -C lz4/lib + # cp -L lz4/lib/liblz4.dylib ./liblz4_64.dylib + # make -C lz4/lib clean + # CFLAGS="-arch i386" CXXFLAGS="-arch i386" LDFLAGS="-arch i386" make -C lz4/lib + # cp -L lz4/lib/liblz4.dylib ./liblz4_32.dylib + # lipo -create ./liblz4_32.dylib ./liblz4_64.dylib -output ./liblz4.dylib + # cp -v ./liblz4.dylib lz4/lib/$(readlink lz4/lib/liblz4.dylib) + # touch lz4/lib/liblz4 + # make -C lz4/lib install + + + mkdir -p rocksdb || fail "unable to create rocksdb directory" + (cd rocksdb && { + checkout "rocksdb" "$ROCKSDBREMOTE" "$ROCKSDBVERSION" "rocksdb_sharp" + + export CFLAGS + export LDFLAGS + (. ./build_tools/build_detect_platform detected~; { + grep detected~ -e '-DSNAPPY' &> /dev/null || fail "failed to detect snappy, install libsnappy-dev" + grep detected~ -e '-DZLIB' &> /dev/null || fail "failed to detect zlib, install libzlib-dev" + grep detected~ -e '-DGFLAGS' &> /dev/null && fail "gflags detected, see https://github.com/facebook/rocksdb/issues/2310" || true + }) || fail "dependency detection failed" + echo "----- Build 64 bit --------------------------------------------------" + make clean + CFLAGS="${CFLAGS}" PORTABLE=1 make -j$CONCURRENCY shared_lib || fail "64-bit build failed" + strip librocksdb${LIBEXT} + mkdir -p ../../native/amd64 && cp -vL ./librocksdb${LIBEXT} ../../native/amd64/librocksdb${LIBEXT} + #mkdir -p ../../native-${ROCKSDBVERSION}/amd64 && cp -vL ./librocksdb${LIBEXT} ../../native-${ROCKSDBVERSION}/amd64/librocksdb${LIBEXT} + echo "----- Build 32 bit --------------------------------------------------" + make clean + CFLAGS="${CFLAGS} -m32" PORTABLE=1 make -j$CONCURRENCY shared_lib || fail "32-bit build failed" + strip librocksdb${LIBEXT} + mkdir -p ../../native/i386 && cp -vL ./librocksdb${LIBEXT} ../../native/i386/librocksdb${LIBEXT} + mkdir -p ../../native-${ROCKSDBVERSION}/i386 && cp -vL ./librocksdb${LIBEXT} ../../native-${ROCKSDBVERSION}/i386/librocksdb${LIBEXT} + + + }) || fail "rocksdb build failed" +fi + + +echo -n ${ROCKSDBSHARPVNUM} > ${BASEDIR}/../version +echo -n ${ROCKSDBVNUM} > ${BASEDIR}/../rocksdbversion + diff --git a/native-build/rocksdb.thirdparty.inc.patch b/native-build/rocksdb.thirdparty.inc.patch new file mode 100644 index 0000000..bf29ec2 --- /dev/null +++ b/native-build/rocksdb.thirdparty.inc.patch @@ -0,0 +1,34 @@ +diff --git a/thirdparty.inc b/thirdparty.inc +index 46da30c..a5eb8ee 100644 +--- a/thirdparty.inc ++++ b/thirdparty.inc +@@ -19,10 +19,10 @@ set (THIRDPARTY_LIBS "") # Initialization, don't touch + # + # Edit these 4 lines to define paths to GFLAGS + # +-set(GFLAGS_HOME $ENV{THIRDPARTY_HOME}/Gflags.Library) +-set(GFLAGS_INCLUDE ${GFLAGS_HOME}/build/native/include) +-set(GFLAGS_LIB_DEBUG ${GFLAGS_HOME}/lib/native/debug/amd64/gflags.lib) +-set(GFLAGS_LIB_RELEASE ${GFLAGS_HOME}/lib/native/retail/amd64/gflags.lib) ++set(GFLAGS_HOME ../gflags/build) ++set(GFLAGS_INCLUDE ${GFLAGS_HOME}/include) ++set(GFLAGS_LIB_DEBUG ../${GFLAGS_HOME}/lib/Debug/gflags) ++set(GFLAGS_LIB_RELEASE ../${GFLAGS_HOME}/lib/Release/gflags) + + # ================================================== GFLAGS ================================================== + # +@@ -63,10 +63,10 @@ endif () + # + # Edit these 4 lines to define paths to Snappy + # +-set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library) +-set(SNAPPY_INCLUDE ${SNAPPY_HOME}/build/native/inc/inc) +-set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/lib/native/debug/amd64/snappy.lib) +-set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/lib/native/retail/amd64/snappy.lib) ++set(SNAPPY_HOME ../snappy) ++set(SNAPPY_INCLUDE ${SNAPPY_HOME}/build) ++set(SNAPPY_LIB_DEBUG ../${SNAPPY_HOME}/build/Debug/libsnappy) ++set(SNAPPY_LIB_RELEASE ../${SNAPPY_HOME}/build/Release/libsnappy) + + # + # Don't touch these lines diff --git a/tests/RocksDbSharpTest/RocksDbSharpTest.csproj b/tests/RocksDbSharpTest/RocksDbSharpTest.csproj index 12c72f5..064e5eb 100644 --- a/tests/RocksDbSharpTest/RocksDbSharpTest.csproj +++ b/tests/RocksDbSharpTest/RocksDbSharpTest.csproj @@ -1,20 +1,25 @@  - netcoreapp2.0 + net6.0 True - PackageReference + latest - - - - + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + -