Read Zarr Segment Index Files

conf/messages

@@ -270,6 +270,8 @@ mesh.file.readVersion.failed=Failed to read format version from file “{0}”
 mesh.file.readMappingName.failed=Failed to read mapping name from mesh file “{0}”
 mesh.meshFileName.required=Trying to load mesh from mesh file, but mesh file name was not supplied.
 
+segmentIndexFile.notFound=Could not find requested segment index file
+
 task.create.noTasks=Zero tasks were requested
 task.create.failed=Failed to create Task
 task.create.limitExceeded=Cannot create more than 1000 tasks in one request.

unreleased_changes/8711.md

@@ -0,0 +1,5 @@
+### Added
+- Segment index files can now also be read from the new zarr3-based format, and from remote object storage.
+
+### Fixed
+- Fixed a bug where segment index files for segmentation layers with no mag1 would be read incorrectly.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

@@ -4,13 +4,24 @@ import org.apache.pekko.actor.ActorSystem
 import com.google.inject.AbstractModule
 import com.google.inject.name.Names
 import com.scalableminds.webknossos.datastore.services._
+import com.scalableminds.webknossos.datastore.services.mapping.{
+  AgglomerateService,
+  Hdf5AgglomerateService,
+  MappingService,
+  ZarrAgglomerateService
+}
 import com.scalableminds.webknossos.datastore.services.mesh.{
   AdHocMeshServiceHolder,
   Hdf5MeshFileService,
   MeshFileService,
   NeuroglancerPrecomputedMeshFileService,
   ZarrMeshFileService
 }
+import com.scalableminds.webknossos.datastore.services.segmentindex.{
+  Hdf5SegmentIndexFileService,
+  SegmentIndexFileService,
+  ZarrSegmentIndexFileService
+}
 import com.scalableminds.webknossos.datastore.services.uploading.UploadService
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptorService}
 
@@ -38,6 +49,9 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[AgglomerateService]).asEagerSingleton()
     bind(classOf[ZarrAgglomerateService]).asEagerSingleton()
     bind(classOf[Hdf5AgglomerateService]).asEagerSingleton()
+    bind(classOf[SegmentIndexFileService]).asEagerSingleton()
+    bind(classOf[ZarrSegmentIndexFileService]).asEagerSingleton()
+    bind(classOf[Hdf5SegmentIndexFileService]).asEagerSingleton()
     bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton()
     bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton()
     bind(classOf[ChunkCacheService]).asEagerSingleton()

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala

@@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.datastore.services.mesh.{AdHocMeshRequest, AdHocMeshService, AdHocMeshServiceHolder}
 import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.services.mapping.MappingService
 import play.api.i18n.Messages
 import play.api.libs.json.Json
 import play.api.mvc.{AnyContent, _}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/SegmentStatistics.scala

@@ -35,8 +35,10 @@ trait SegmentStatistics extends ProtoGeometryImplicits with FoxImplicits {
       implicit ec: ExecutionContext): Fox[Long] =
     for {
       bucketPositionsProtos: Set[Vec3IntProto] <- getBucketPositions(segmentId, mag)
-      bucketPositionsInMag = bucketPositionsProtos.map(vec3IntFromProto)
-      (bucketBoxes, elementClass) <- getDataForBucketPositions(bucketPositionsInMag.toSeq, mag, additionalCoordinates)
+      bucketPositionsInRequestedMag = bucketPositionsProtos.map(vec3IntFromProto)
+      (bucketBoxes, elementClass) <- getDataForBucketPositions(bucketPositionsInRequestedMag.toSeq,
+                                                               mag,
+                                                               additionalCoordinates)
       counts <- Fox.serialCombined(bucketBoxes.toList) {
         case Full(bucketBytes) =>
           tryo(

webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DatasetLayerAttachments.scala

@@ -38,7 +38,16 @@ object LayerAttachmentType extends ExtendedEnumeration {
 case class LayerAttachment(name: String,
                            path: URI,
                            dataFormat: LayerAttachmentDataformat.LayerAttachmentDataformat,
-                           credentialId: Option[String] = None)
+                           credentialId: Option[String] = None) {
+  // Warning: throws! Use inside of tryo
+  def localPath: Path = {
+    if (path.getScheme.nonEmpty && path.getScheme != "file") {
+      throw new Exception(
+        "Trying to open non-local hdf5 file. Hdf5 files are only supported on the datastore-local file system.")
+    }
+    Path.of(path)
+  }
+}
 
 object LayerAttachment {
   implicit val jsonFormat: Format[LayerAttachment] = Json.format[LayerAttachment]

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/Hdf5HashedArrayUtils.scala → webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ArrayArtifactHashing.scala

@@ -4,14 +4,13 @@ import org.apache.commons.codec.digest.MurmurHash3
 
 import java.nio.ByteBuffer
 
-trait Hdf5HashedArrayUtils {
-
-  val hdf5FileExtension = "hdf5"
+trait ArrayArtifactHashing {
 
   def getHashFunction(name: String): Long => Long = name match {
     case "identity" => identity
     case "murmurhash3_x64_128" =>
       x: Long =>
         Math.abs(MurmurHash3.hash128x64(ByteBuffer.allocate(8).putLong(x).array())(1))
   }
+
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -14,6 +14,7 @@ import com.typesafe.scalalogging.LazyLogging
 import com.scalableminds.util.tools.{Box, Empty, Full}
 import ucar.ma2.{Array => MultiArray}
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
 
 import java.nio.file.Path
 import scala.concurrent.ExecutionContext

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala

@@ -2,6 +2,7 @@ package com.scalableminds.webknossos.datastore.services
 
 import java.nio.file.Paths
 import com.scalableminds.webknossos.datastore.DataStoreConfig
+import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
 import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
 
 import javax.inject.Inject

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/VoxelyticsZarrArtifactUtils.scala

@@ -0,0 +1,20 @@
+package com.scalableminds.webknossos.datastore.services
+
+import play.api.libs.json.{JsLookupResult, JsResult, JsValue}
+
+trait VoxelyticsZarrArtifactUtils {
+
+  val FILENAME_ZARR_JSON = "zarr.json"
+
+  private val keyAttributes = "attributes"
+  private val keyVx = "voxelytics"
+  private val keyFormatVersion = "artifact_schema_version"
+  private val keyArtifactAttrs = "artifact_attributes"
+
+  protected def readArtifactSchemaVersion(zarrGroupJson: JsValue): JsResult[Long] =
+    (zarrGroupJson \ keyAttributes \ keyVx \ keyFormatVersion).validate[Long]
+
+  protected def lookUpArtifactAttributes(zarrGroupJson: JsValue): JsLookupResult =
+    zarrGroupJson \ keyAttributes \ keyVx \ keyArtifactAttrs
+
+}