Test what changed

.github/workflows/trigger-build.yml

@@ -0,0 +1,16 @@
+name: Trigger RocksDB build on nethermind repo
+
+on: 
+  push:
+    branches:
+    - master
+
+jobs:
+  bls-building:
+    name: Building RocksDB
+    runs-on: ubuntu-latest
+    steps:
+    - name: Send dispatch event to nethermind repo
+      run: |
+        curl -v -X POST -u "${{ secrets.REPOSITORY_DISPATCH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" --data '{"event_type":"rocksdb_build"}' https://github.com/api/repos/nethermindeth/nethermind/dispatches
+

.gitignore

@@ -179,14 +179,13 @@ $RECYCLE.BIN/
 # SASS Compiler cache
 .sass-cache
 
+# Rider
+.idea/
+
 # Visual Studio 2014 CTP
 **/*.sln.ide
 .vs/
 
-amd64/rocksdb.dll
-native/**/*.dll
-native/
-native*/
 *.xz
 *.zip
 *.userprefs

RocksDbNative/RocksDbNative.csproj

@@ -1,8 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
-  <Import Project="..\Versions.targets.include"/>
+  <Import Project="..\Versions.targets.include" />
   <PropertyGroup>
     <Title>RocksDbNative</Title>
-    <TargetFrameworks>netstandard1.6;net40;net45</TargetFrameworks>
     <Version>$(RocksDbVersion).$(RocksDbNativeBuild)</Version>
     <AssemblyVersion>$(Version)</AssemblyVersion>
     <FileVersion>$(Version)</FileVersion>
@@ -18,51 +17,59 @@
     <Copyright>Copyright 2016</Copyright>
     <PackageReleaseNotes>Native RocksDb Binaries (Install RocksDbSharp package for managed wraper)</PackageReleaseNotes>
     <GeneratePackageOnBuild>False</GeneratePackageOnBuild>
+    <TargetFramework>netstandard2.1</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
   <!-- "DispatchToInnerBuilds" will cause this to run in the outer build (i.e. once regardless of number of frameworks) -->
   <!--  Condition="!Exists('rocksdb-$(RocksDbVersion)\runtimes\win-x64\native\rocksdb.dll') and !Exists('$(BaseIntermediateOutputPath)\rocksdb-v$(RocksDbVersion)-win-x64.zip')" -->
-  <Target Name="DownloadNativeLibs" BeforeTargets="DispatchToInnerBuilds">
-    <!-- Here we download the native libraries from the rocksdb-sharp-native project releases according to the version number, if they don't exist -->
-    <DownloadFile
-      Condition="!Exists('$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\%(NativeLib.Lib)') And !Exists('$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip')"
-      SourceUrl="https://github.com/warrenfalk/rocksdb-sharp-native/releases/download/v$(RocksDbVersion)/rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip"
-      DestinationFolder="$(BaseIntermediateOutputPath)">
-    </DownloadFile>
-    <Unzip
-      Condition="Exists('$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip') And !Exists('$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\%(NativeLib.Lib)')"
-      SourceFiles="$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip"
-      DestinationFolder="$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\">
-    </Unzip>
-  </Target>
+  <!--<Target Name="DownloadNativeLibs" BeforeTargets="DispatchToInnerBuilds">-->
+    <!--&lt;!&ndash; Here we download the native libraries from the rocksdb-sharp-native project releases according to the version number, if they don't exist &ndash;&gt;-->
+    <!--<DownloadFile Condition="!Exists('$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\%(NativeLib.Lib)') And !Exists('$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip')" SourceUrl="https://github.com/warrenfalk/rocksdb-sharp-native/releases/download/v$(RocksDbVersion)/rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip" DestinationFolder="$(BaseIntermediateOutputPath)">-->
+    <!--</DownloadFile>-->
+    <!--<Unzip Condition="Exists('$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip') And !Exists('$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\%(NativeLib.Lib)')" SourceFiles="$(BaseIntermediateOutputPath)rocksdb-v$(RocksDbVersion)-%(NativeLib.RuntimeId).zip" DestinationFolder="$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(NativeLib.RuntimeId)\native\">-->
+    <!--</Unzip>-->
+  <!--</Target>-->
   <ItemGroup>
     <!-- Define the native libraries here -->
     <!-- The "Include" attribute is a required attribute in ItemGroups outside of Targets, but I don't know why -->
-    <NativeLib Include="%(RuntimeId)-%(Lib)" RuntimeId="win-x64" Lib="rocksdb.dll" />
-    <NativeLib Include="%(RuntimeId)-%(Lib)" RuntimeId="linux-x64" Lib="librocksdb.so" />
-    <NativeLib Include="%(RuntimeId)-%(Lib)" RuntimeId="osx-x64" Lib="librocksdb.dylib" />
+    <!--<NativeLib Include="win-x64-rocksdb.dll" RuntimeId="win-x64" Lib="rocksdb.dll" />-->
+    <!--<NativeLib Include="linux-x64-librocksdb.so" RuntimeId="linux-x64" Lib="librocksdb.so" />-->
+    <!--<NativeLib Include="osx-x64-librocksdb.dylib" RuntimeId="osx-x64" Lib="librocksdb.dylib" />-->
+
+    <!--<Content Include="@(NativeLib -> '.\runtimes\%(RuntimeId)\native\%(Lib)')">-->
+      <!--<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>-->
+    <!--</Content>-->
     <!-- This "Content" is applied for all "NativeLib" and ensures that it is packaged into the correct folder -->
     <!-- This also will cause it to be copied to the output directory, which was so far the only way to get it
          to work correctly when referenced using a ProjectReference.  When packaged as a nupkg and referenced
-         with PackageReference, then these go to the runtimes folders in the nuget cache, which is enough for 
+         with PackageReference, then these go to the runtimes folders in the nuget cache, which is enough for
          standard consumers, but framework consumers get them copied from the nuget cache to the output via
          the .targets files for those frameworks.-->
-    <Content
-      Include="@(NativeLib -> '$(BaseIntermediateOutputPath)rocksdb-$(RocksDbVersion)\runtimes\%(RuntimeId)\native\%(Lib)')"
-      PackagePath="runtimes\%(RuntimeId)\native\%(Lib)"
-      Link="runtimes\%(RuntimeId)\native\%(Lib)">
-      <Pack>true</Pack>
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <!-- These target files take care of copying from the nuget cache area into the output folder as required 
+    <Compile Remove="build\**" />
+    <EmbeddedResource Remove="build\**" />
+    <!-- These target files take care of copying from the nuget cache area into the output folder as required
          by the .Net Framework frameworks -->
     <None Remove="build\**" />
-    <None Include="build\net40\RocksDbNative.targets">
-      <Pack>true</Pack>
-      <PackagePath>build/net40</PackagePath>
-    </None>
-    <None Include="build\net45\RocksDbNative.targets">
-      <Pack>true</Pack>
-      <PackagePath>build/net45</PackagePath>
-    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="runtimes\linux-arm64\native\librocksdb.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="runtimes\linux-arm\native\librocksdb.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="runtimes\linux-x64\native\librocksdb.so">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="runtimes\osx-x64\native\librocksdb.dylib">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="runtimes\osx-arm64\native\librocksdb.dylib">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="runtimes\win-x64\native\rocksdb.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 </Project>

RocksDbSharp/AutoNativeImport.cs

@@ -192,7 +192,7 @@ public static string GetArchName(Architecture arch)
                 case Architecture.X86:
                     return "i386";
                 case Architecture.X64:
-                    return "amd64";
+                    return "x64";
                 default:
                     return arch.ToString().ToLower();
             }
@@ -368,6 +368,20 @@ public static T Import<T>(INativeLibImporter importer, string libName, string ve
                 "",
             };
 
+            HashSet<string> basePaths = new();
+            void TryAddBasePath(string path)
+            {
+                if (path is not null)
+                {
+                    try { basePaths.Add(path); }
+                    catch
+                    {
+                        /* Ignore */
+                    }
+                }
+            }
+
+
             // If the RocksDbNative package is referenced, then dynamically load it here so that it can tell us where the native libraries are
             string nativeCodeBase = null;
             try
@@ -379,18 +393,24 @@ public static T Import<T>(INativeLibImporter importer, string libName, string ve
                 var getCodeBase = getCodeBaseMethod.CreateDelegate<Func<string>>();
                 nativeCodeBase = getCodeBase();
             }
-            catch (Exception)
+            catch { /* Ignore */ }
+
+            //Some paths might throw NotSupportedException when used from single file deployment. We could test for that, but we can also just ignore it
+            TryAddBasePath(nativeCodeBase);
+            TryAddBasePath(Directory.GetCurrentDirectory());
+            TryAddBasePath(Path.GetDirectoryName(UriToPath(AppContext.BaseDirectory)));
+            TryAddBasePath(Path.GetDirectoryName(UriToPath(Transitional.CurrentFramework.GetBaseDirectory())));
+            TryAddBasePath(Path.GetDirectoryName(UriToPath(Assembly.GetEntryAssembly()?.Location)));
+            TryAddBasePath(Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location));
+            TryAddBasePath(Path.GetDirectoryName(UriToPath(typeof(PosixImporter).GetTypeInfo().Assembly.Location)));
+            TryAddBasePath(Path.GetDirectoryName(typeof(PosixImporter).GetTypeInfo().Assembly.Location));
+
+            // for Apple ARM64 - M1
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
             {
-            }
+                TryAddBasePath("/opt/homebrew/lib");
+            } 
 
-            var basePaths = new string[] {
-                nativeCodeBase,
-                Path.GetDirectoryName(UriToPath(Transitional.CurrentFramework.GetBaseDirectory())),
-                Path.GetDirectoryName(UriToPath(Assembly.GetEntryAssembly()?.CodeBase)),
-                Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location),
-                Path.GetDirectoryName(UriToPath(typeof(PosixImporter).GetTypeInfo().Assembly.CodeBase)),
-                Path.GetDirectoryName(typeof(PosixImporter).GetTypeInfo().Assembly.Location),
-            };
             var search = basePaths
                 .Where(p => p != null)
                 .Distinct()

RocksDbSharp/Native.Load.cs

@@ -15,7 +15,7 @@ static Native()
         {
             if (RuntimeInformation.ProcessArchitecture == Architecture.X86 && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 throw new RocksDbSharpException("Rocksdb on windows is not supported for 32 bit applications");
-            Instance = NativeImport.Auto.Import<Native>("rocksdb", "6.2.2", true);
+            Instance = NativeImport.Auto.Import<Native>("rocksdb", "6.4.6", true);
         }
 
         public Native()

RocksDbSharp/Native.Marshaled.cs

@@ -1,10 +1,11 @@
-/*
+/*
     The functions in this file provide some wrappers around the lowest level C API to aid in marshalling.
     This is kept separate so that the lowest level imports can be kept as close as possible to c.h from rocksdb.
     See Native.Raw.cs for more information.
 */
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
 using Transitional;
@@ -155,7 +156,37 @@ public byte[] rocksdb_get(
             rocksdb_free(resultPtr);
             return result;
         }
-
+
+        public unsafe Span<byte> rocksdb_get_span(
+            IntPtr db,
+            IntPtr read_options,
+            byte[] key,
+            long keyLength,
+            out IntPtr errptr,
+            ColumnFamilyHandle cf = null)
+        {
+            UIntPtr skLength = (UIntPtr)keyLength;
+            var resultPtr = cf == null
+                ? rocksdb_get(db, read_options, key, skLength, out UIntPtr valueLength, out errptr)
+                : rocksdb_get_cf(db, read_options, cf.Handle, key, skLength, out valueLength, out errptr);
+            if (errptr != IntPtr.Zero)
+                return null;
+            if (resultPtr == IntPtr.Zero)
+                return null;
+            Span<byte> span = new Span<byte>((void*)resultPtr, (int)valueLength);
+            //MemoryMarshal.GetReference(span);
+
+
+            return span;
+        }
+
+        public unsafe void rocksdb_release_span(in Span<byte> span)
+        {
+            ref byte ptr = ref MemoryMarshal.GetReference(span);
+            IntPtr intPtr = new IntPtr(Unsafe.AsPointer(ref ptr));
+            rocksdb_free(intPtr);
+        }
+
         /// <summary>
         /// Executes a multi_get with automatic marshalling
         /// </summary>

RocksDbSharp/Native.Wrap.cs

@@ -44,6 +44,26 @@ public void rocksdb_put(
             if (errptr != IntPtr.Zero)
                 throw new RocksDbException(errptr);
         }
+
+        public unsafe void rocksdb_put(
+            IntPtr db,
+            IntPtr writeOptions,
+            byte* key,
+            long keyLength,
+            byte* value,
+            long valueLength,
+            ColumnFamilyHandle cf)
+        {
+            IntPtr errptr;
+            UIntPtr sklength = (UIntPtr)keyLength;
+            UIntPtr svlength = (UIntPtr)valueLength;
+            if (cf == null)
+                rocksdb_put(db, writeOptions, key, sklength, value, svlength, out errptr);
+            else
+                rocksdb_put_cf(db, writeOptions, cf.Handle, key, sklength, value, svlength, out errptr);
+            if (errptr != IntPtr.Zero)
+                throw new RocksDbException(errptr);
+        }
 
 
         public string rocksdb_get(
@@ -89,6 +109,19 @@ public byte[] rocksdb_get(
                 throw new RocksDbException(errptr);
             return result;
         }
+
+        public Span<byte> rocksdb_get_span(
+            IntPtr db,
+            IntPtr read_options,
+            byte[] key,
+            long keyLength = 0,
+            ColumnFamilyHandle cf = null)
+        {
+            var result = rocksdb_get_span(db, read_options, key, keyLength == 0 ? key.Length : keyLength, out IntPtr errptr, cf);
+            if (errptr != IntPtr.Zero)
+                throw new RocksDbException(errptr);
+            return result;
+        }
 
         public System.Collections.Generic.KeyValuePair<string, string>[] rocksdb_multi_get(
             IntPtr db,