Add unit tests for native bucket scanner, also check it in data/health

.github/workflows/build_test_deploy.yml

@@ -233,19 +233,19 @@ jobs:
       - name: Run webknossos smoke test
         uses: ./.github/actions/retry
         with:
-          run: curl -v --fail "http://localhost:9000/api/health"
+          run: curl -v --fail-with-body "http://localhost:9000/api/health"
           retries: 20
           retry_delay_seconds: 5
       - name: Run webknossos-datastore smoke test
         uses: ./.github/actions/retry
         with:
-          run: curl -v --fail "http://localhost:9090/data/health"
+          run: curl -v --fail-with-body "http://localhost:9090/data/health"
           retries: 20
           retry_delay_seconds: 5
       - name: Run webknossos-tracingstore smoke test
         uses: ./.github/actions/retry
         with:
-          run: curl -v --fail "http://localhost:9050/tracings/health"
+          run: curl -v --fail-with-body "http://localhost:9050/tracings/health"
           retries: 20
           retry_delay_seconds: 5
       - name: Stop webknossos, datastore and tracingstore

test/backend/BucketScannerTestSuite.scala

@@ -0,0 +1,97 @@
+package backend
+
+import com.scalableminds.webknossos.datastore.helpers.NativeBucketScanner
+import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass}
+import org.scalatestplus.play.PlaySpec
+
+class BucketScannerTestSuite extends PlaySpec {
+  "NativeBucketScanner" should {
+    "collect segment ids in a byte array with ElementClass uint16" in {
+      val elementClass = ElementClass.uint16
+      // little endian uint16 representation of 2, 4, 500, 500
+      val array = Array[Byte](2, 0, 4, 0, 244.toByte, 1, 244.toByte, 1)
+      val scanner = new NativeBucketScanner()
+      val segmentIds = scanner.collectSegmentIds(array,
+                                                 ElementClass.bytesPerElement(elementClass),
+                                                 ElementClass.isSigned(elementClass),
+                                                 skipZeroes = false)
+      assert(segmentIds.sorted.sameElements(Array[Long](2, 4, 500)))
+    }
+
+    "collect segment ids in a byte array with ElementClass uint32" in {
+      val elementClass = ElementClass.uint32
+      // little endian uint32 representation of 2, 4, 500, 500
+      val array = Array[Byte](2, 0, 0, 0, 4, 0, 0, 0, 244.toByte, 1, 0, 0, 244.toByte, 1, 0, 0)
+      val scanner = new NativeBucketScanner()
+      val segmentIds = scanner.collectSegmentIds(array,
+                                                 ElementClass.bytesPerElement(elementClass),
+                                                 ElementClass.isSigned(elementClass),
+                                                 skipZeroes = false)
+      assert(segmentIds.sorted.sameElements(Array[Long](2, 4, 500)))
+    }
+
+    "skip zeroes in collectSegmentIds if requested" in {
+      val elementClass = ElementClass.uint16
+      // little endian uint16 representation of 2, 4, 500, 500, 0
+      val array = Array[Byte](2, 0, 4, 0, 244.toByte, 1, 244.toByte, 1, 0, 0)
+      val scanner = new NativeBucketScanner()
+      val segmentIds = scanner.collectSegmentIds(array,
+                                                 ElementClass.bytesPerElement(elementClass),
+                                                 ElementClass.isSigned(elementClass),
+                                                 skipZeroes = false)
+      assert(segmentIds.sorted.sameElements(Array[Long](0, 2, 4, 500)))
+
+      val segmentIds2 = scanner.collectSegmentIds(array,
+                                                  ElementClass.bytesPerElement(elementClass),
+                                                  ElementClass.isSigned(elementClass),
+                                                  skipZeroes = true)
+      assert(segmentIds2.sorted.sameElements(Array[Long](2, 4, 500)))
+    }
+
+    "count segment voxels correctly in a byte array with ElementClass uint32" in {
+      val elementClass = ElementClass.uint32
+      // little endian uint32 representation of 2, 4, 500, 500
+      val array = Array[Byte](2, 0, 0, 0, 4, 0, 0, 0, 244.toByte, 1, 0, 0, 244.toByte, 1, 0, 0)
+      val scanner = new NativeBucketScanner()
+      val voxelCount = scanner.countSegmentVoxels(array,
+                                                  ElementClass.bytesPerElement(elementClass),
+                                                  ElementClass.isSigned(elementClass),
+                                                  segmentId = 500)
+      assert(voxelCount == 2)
+      val voxelCount2 = scanner.countSegmentVoxels(array,
+                                                   ElementClass.bytesPerElement(elementClass),
+                                                   ElementClass.isSigned(elementClass),
+                                                   segmentId = 501)
+      assert(voxelCount2 == 0)
+    }
+
+    "find bounding box of segment correctly in a byte array with ElementClass uint16" in {
+      val elementClass = ElementClass.uint16
+      val bytesPerBucket = ElementClass.bytesPerElement(elementClass) * scala.math
+        .pow(DataLayer.bucketLength, 3)
+        .intValue
+      val array = Array.fill[Byte](bytesPerBucket)(0)
+      array(ElementClass.bytesPerElement(elementClass) * (DataLayer.bucketLength + 5)) = 1
+      array(ElementClass.bytesPerElement(elementClass) * (DataLayer.bucketLength + 8)) = 1
+      val scanner = new NativeBucketScanner()
+      val boundingBox = scanner.extendSegmentBoundingBox(
+        array,
+        ElementClass.bytesPerElement(elementClass),
+        ElementClass.isSigned(elementClass),
+        DataLayer.bucketLength,
+        1,
+        0,
+        0,
+        0,
+        Int.MaxValue,
+        Int.MaxValue,
+        Int.MaxValue,
+        Int.MinValue,
+        Int.MinValue,
+        Int.MinValue
+      )
+      assert(boundingBox.sameElements(Array[Long](5, 1, 0, 8, 1, 0)))
+    }
+
+  }
+}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala

@@ -2,8 +2,11 @@ package com.scalableminds.webknossos.datastore.controllers
 
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.helpers.NativeBucketScanner
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
 import com.scalableminds.webknossos.datastore.services.ApplicationHealthService
 import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore
+import net.liftweb.common.Box.tryo
 
 import javax.inject.Inject
 import play.api.mvc.{Action, AnyContent}
@@ -23,11 +26,31 @@ class Application @Inject()(redisClient: DataStoreRedisStore, applicationHealthS
         _ <- redisClient.checkHealth
         afterRedis = Instant.now
         _ <- Fox.bool2Fox(applicationHealthService.getRecentProblem().isEmpty) ?~> "Java Internal Errors detected"
+        _ <- testNativeBucketScanner.toFox ?~> "NativeBucketScanner error"
         _ = logger.info(
           s"Answering ok for Datastore health check, took ${formatDuration(afterRedis - before)} (Redis at ${redisClient.authority} ${formatDuration(
             afterRedis - before)})")
       } yield Ok("Ok")
     }
   }
 
+  // Test that the NativeBucketScanner works.
+  // The result is stored in a val because we expect that this continues to work if it works on startup.
+  private lazy val testNativeBucketScanner = tryo {
+    val elementClass = ElementClass.uint16
+    // little endian uint16 representation of 2, 4, 500, 500
+    val array = Array[Byte](2, 0, 4, 0, 244.toByte, 1, 244.toByte, 1)
+    val scanner = new NativeBucketScanner()
+    val segmentIds = scanner.collectSegmentIds(array,
+                                               ElementClass.bytesPerElement(elementClass),
+                                               ElementClass.isSigned(elementClass),
+                                               skipZeroes = false)
+    val expected = Array[Long](2, 4, 500)
+    if (!segmentIds.sorted.sameElements(expected)) {
+      throw new IllegalStateException(
+        s"NativeBucketScanner did not scan segment ids of test array correctly. Expected ${expected
+          .mkString(",")}, got ${segmentIds.mkString(",")}")
+    }
+  }
+
 }