Fix format in volume annotation download zarr.json

unreleased_changes/8800.md

@@ -0,0 +1,2 @@
+### Fixed
+- Fixed a bug where the zarr.json included in volume annotation zarr downloads would miss the compression codec even though the chunks were compressed.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala

@@ -258,7 +258,9 @@ object Zarr3ArrayHeader extends JsonImplicits {
       )
 
   }
-  def fromDataLayer(dataLayer: DataLayer, mag: Vec3Int): Zarr3ArrayHeader = {
+  def fromDataLayer(dataLayer: DataLayer,
+                    mag: Vec3Int,
+                    additionalCodecs: Seq[CodecConfiguration] = Seq.empty): Zarr3ArrayHeader = {
     val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty))
     val xyzBBounds = Array(
       // Zarr can't handle data sets that don't start at 0, so we extend the shape to include "true" coords
@@ -287,7 +289,7 @@ object Zarr3ArrayHeader extends JsonImplicits {
       codecs = Seq(
         TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)),
         BytesCodecConfiguration(Some("little")),
-      ),
+      ) ++ additionalCodecs,
       storage_transformers = None,
       dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z"))
     )

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala

@@ -95,10 +95,12 @@ trait VolumeDataZipHelper extends WKWDataFormatHelper with ReversionHelper with
     val additionalAxesNames: Seq[String] = dimensionNames.toSeq.drop(1).dropRight(3) // drop channel left, and xyz right
 
     // assume additionalAxes,x,y,z
-    val chunkPathRegex = s"(|.*/)(\\d+|\\d+-\\d+-\\d+)/c\\.(.+)".r
+    // the c. at the beginning of chunk names is optional
+    // (used in "default" chunk encoding, which was used for volume downloads previously)
+    val chunkPathRegex = s"(|.*/)(\\d+|\\d+-\\d+-\\d+)/(c\\.)?(.+)".r
 
     path match {
-      case chunkPathRegex(_, magStr, dimsStr) =>
+      case chunkPathRegex(_, magStr, _, dimsStr) =>
         val dims: Seq[String] = dimsStr.split("\\.").toSeq
         val additionalAxesDims = dims.drop(1).dropRight(3) // drop channel left, and xyz right
         val additionalCoordinates: Seq[AdditionalCoordinate] = additionalAxesNames.zip(additionalAxesDims).map {

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala

@@ -41,7 +41,9 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay
   def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])(
       implicit ec: ExecutionContext): Iterator[NamedStream] = {
 
-    val header = Zarr3ArrayHeader.fromDataLayer(layer, mags.headOption.getOrElse(Vec3Int.ones))
+    val header = Zarr3ArrayHeader.fromDataLayer(layer,
+                                                mags.headOption.getOrElse(Vec3Int.ones),
+                                                additionalCodecs = Seq(compressorConfiguration))
     bucketStream.flatMap {
       case (bucket, data) =>
         val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedElement(data)
@@ -100,7 +102,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay
     val additionalCoordinatesPart =
       additionalCoordinatesFilePath(bucketPosition.additionalCoordinates, additionalAxesSorted)
     val channelPart = 0
-    s"$layerName/${bucketPosition.mag.toMagLiteral(allowScalar = true)}/c$dimensionSeparator$channelPart$dimensionSeparator$additionalCoordinatesPart${bucketPosition.bucketX}$dimensionSeparator${bucketPosition.bucketY}$dimensionSeparator${bucketPosition.bucketZ}"
+    s"$layerName/${bucketPosition.mag.toMagLiteral(allowScalar = true)}/$channelPart$dimensionSeparator$additionalCoordinatesPart${bucketPosition.bucketX}$dimensionSeparator${bucketPosition.bucketY}$dimensionSeparator${bucketPosition.bucketZ}"
   }
 
   private def additionalCoordinatesFilePath(additionalCoordinatesOpt: Option[Seq[AdditionalCoordinate]],
@@ -116,6 +118,15 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay
   private def zarrHeaderFilePath(layerName: String, mag: Vec3Int): String =
     s"$layerName/${mag.toMagLiteral(allowScalar = true)}/${Zarr3ArrayHeader.FILENAME_ZARR_JSON}"
 
+  private lazy val compressorConfiguration =
+    BloscCodecConfiguration(
+      BloscCompressor.defaultCname,
+      BloscCompressor.defaultCLevel,
+      StringCompressionSetting(BloscCodecConfiguration.shuffleSettingFromInt(BloscCompressor.defaultShuffle)),
+      Some(BloscCompressor.defaultTypesize),
+      BloscCompressor.defaultBlocksize
+    )
+
   private lazy val compressor =
     new BloscCompressor(
       Map(

webknossos-tracingstore/conf/tracingstore.latest.routes

@@ -18,7 +18,7 @@ POST        /volume/save
 POST        /volume/:tracingId/initialData                                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(annotationId: ObjectId, tracingId: String, minMag: Option[Int], maxMag: Option[Int])
 POST        /volume/:tracingId/initialDataMultiple                                  @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(annotationId: ObjectId, tracingId: String)
 GET         /volume/:tracingId                                                      @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(tracingId: String, annotationId: ObjectId, version: Option[Long])
-GET         /volume/:tracingId/allDataZip                                           @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(tracingId: String, annotationId: Option[ObjectId], version: Option[Long], volumeDataZipFormat: String, voxelSize: Option[String], voxelSizeUnit: Option[String])
+GET         /volume/:tracingId/allDataZip                                           @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(tracingId: String, annotationId: Option[ObjectId], version: Option[Long], volumeDataZipFormat: String, voxelSizeFactor: Option[String], voxelSizeUnit: Option[String])
 POST        /volume/:tracingId/data                                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(tracingId: String, annotationId: ObjectId)
 POST        /volume/:tracingId/adHocMesh                                            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(tracingId: String)
 POST        /volume/:tracingId/fullMesh.stl                                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(tracingId: String)