Skip to content

Commit a68ef77

Browse files
MichaelBuessemeyerMichaelBuessemeyer
authored
Merge branch 'master' into allow-passing-ds_id-on-manual-upload
2 parents 0f471fa + f729741 commit a68ef77

File tree

21 files changed

+433
-203
lines changed

21 files changed

+433
-203
lines changed

‎CHANGELOG.unreleased.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
1515
- The opacity of meshes can be adjusted using the 'Change Segment Color' context menu entry in the segments tab. [#8443](https://github.com/scalableminds/webknossos/pull/8443)
1616
- The maximum available storage of an organization is now enforced during upload. [#8385](https://github.com/scalableminds/webknossos/pull/8385)
1717
- Performance improvements for volume annotation save requests. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
18+
- Performance improvements for segment statistics (volume + bounding box in context menu). [#8469](https://github.com/scalableminds/webknossos/pull/8469)
1819

1920
### Changed
2021
- Added a parameter to the reserve manual upload route allowing to make the request fail if the name is already taken. Moreover, the new dataset's id and directory name are returned in the response. [#8476](https://github.com/scalableminds/webknossos/pull/8476)
@@ -31,6 +32,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
3132
- Fixed a bug where segment statistics would sometimes be wrong in case of an on-disk segmentation fallback layer with segment index file. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
3233
- Fixed a bug where sometimes outdated segment statistics would be displayed. [#8460](https://github.com/scalableminds/webknossos/pull/8460)
3334
- Fixed a bug where the annotation list would sometimes load very long if you have many annotations. [#8498](https://github.com/scalableminds/webknossos/pull/8498)
35+
- Fixed a bug where outbound zarr streaming would contain a typo in the zarr header dimension_separator field. [#8510](https://github.com/scalableminds/webknossos/pull/8510)
3436

3537
### Removed
3638

‎util/src/main/scala/com/scalableminds/util/tools/Fox.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ object Fox extends FoxImplicits {
8888
}
8989

9090
// run serially, return individual results in list of box
91-
def serialSequenceBox[A, B](l: List[A])(f: A => Fox[B])(implicit ec: ExecutionContext): Future[List[Box[B]]] = {
91+
def serialSequenceBox[A, B](l: Seq[A])(f: A => Fox[B])(implicit ec: ExecutionContext): Future[List[Box[B]]] = {
9292
def runNext(remaining: List[A], results: List[Box[B]]): Future[List[Box[B]]] =
9393
remaining match {
9494
case head :: tail =>
@@ -99,7 +99,7 @@ object Fox extends FoxImplicits {
9999
case Nil =>
100100
Future.successful(results.reverse)
101101
}
102-
runNext(l, Nil)
102+
runNext(l.toList, Nil)
103103
}
104104

105105
def sequence[T](l: List[Fox[T]])(implicit ec: ExecutionContext): Future[List[Box[T]]] =

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class DataStoreModule extends AbstractModule {
2525
bind(classOf[AgglomerateService]).asEagerSingleton()
2626
bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton()
2727
bind(classOf[ApplicationHealthService]).asEagerSingleton()
28-
bind(classOf[DatasetErrorLoggingService]).asEagerSingleton()
28+
bind(classOf[DSDatasetErrorLoggingService]).asEagerSingleton()
2929
bind(classOf[MeshFileService]).asEagerSingleton()
3030
}
3131
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class DataSourceController @Inject()(
4343
connectomeFileService: ConnectomeFileService,
4444
segmentIndexFileService: SegmentIndexFileService,
4545
storageUsageService: DSUsedStorageService,
46-
datasetErrorLoggingService: DatasetErrorLoggingService,
46+
datasetErrorLoggingService: DSDatasetErrorLoggingService,
4747
exploreRemoteLayerService: ExploreRemoteLayerService,
4848
uploadService: UploadService,
4949
composeService: ComposeService,

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,16 @@ package com.scalableminds.webknossos.datastore.dataformats
33
import com.scalableminds.util.accesscontext.TokenContext
44
import com.scalableminds.util.tools.Fox
55
import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction
6+
import net.liftweb.common.Box
67

78
import scala.concurrent.ExecutionContext
89

910
trait BucketProvider {
1011
def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]]
12+
13+
def loadMultiple(readInstructions: Seq[DataReadInstruction])(implicit ec: ExecutionContext,
14+
tc: TokenContext): Fox[Seq[Box[Array[Byte]]]] =
15+
Fox.serialSequenceBox(readInstructions) { readInstruction =>
16+
load(readInstruction)
17+
}
1118
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/ZarrHeader.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ object ZarrHeader extends JsonImplicits {
9898
"compressor" -> zarrHeader.compressor,
9999
"filters" -> None,
100100
"shape" -> zarrHeader.shape,
101-
"dimension_seperator" -> zarrHeader.dimension_separator
101+
"dimension_separator" -> zarrHeader.dimension_separator
102102
)
103103
}
104104
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/NativeBucketScanner.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,24 @@ class NativeBucketScanner() {
88
bytesPerElement: Int,
99
isSigned: Boolean,
1010
skipZeroes: Boolean): Array[Long]
11+
12+
@native def countSegmentVoxels(bucketBytes: Array[Byte],
13+
bytesPerElement: Int,
14+
isSigned: Boolean,
15+
segmentId: Long): Long
16+
17+
@native def extendSegmentBoundingBox(bucketBytes: Array[Byte],
18+
bytesPerElement: Int,
19+
isSigned: Boolean,
20+
bucketLength: Int,
21+
segmentId: Long,
22+
bucketTopLeftX: Int,
23+
bucketTopLeftY: Int,
24+
bucketTopLeftZ: Int,
25+
existingBBoxTopLeftX: Int,
26+
existingBBoxTopLeftY: Int,
27+
existingBBoxTopLeftZ: Int,
28+
existingBBoxBottomRightX: Int,
29+
existingBBoxBottomRightY: Int,
30+
existingBBoxBottomRightZ: Int): Array[Int]
1131
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/SegmentStatistics.scala‎

Lines changed: 69 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package com.scalableminds.webknossos.datastore.helpers
33
import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
44
import com.scalableminds.util.tools.{Fox, FoxImplicits}
55
import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
6-
import com.scalableminds.webknossos.datastore.models.datasource.DataLayer
7-
import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, SegmentInteger}
6+
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass}
7+
import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
8+
import net.liftweb.common.{Box, Empty, Failure, Full}
9+
import net.liftweb.common.Box.tryo
810
import play.api.libs.json.{Json, OFormat}
911

1012
import scala.concurrent.ExecutionContext
@@ -19,35 +21,44 @@ object SegmentStatisticsParameters {
1921

2022
trait SegmentStatistics extends ProtoGeometryImplicits with FoxImplicits {
2123

22-
def calculateSegmentVolume(segmentId: Long,
23-
mag: Vec3Int,
24-
additionalCoordinates: Option[Seq[AdditionalCoordinate]],
25-
getBucketPositions: (Long, Vec3Int) => Fox[Set[Vec3IntProto]],
26-
getTypedDataForBucketPosition: (
27-
Vec3Int,
28-
Vec3Int,
29-
Option[Seq[AdditionalCoordinate]]) => Fox[Array[SegmentInteger]])(
24+
protected def bucketScanner: NativeBucketScanner
25+
26+
def calculateSegmentVolume(
27+
segmentId: Long,
28+
mag: Vec3Int,
29+
additionalCoordinates: Option[Seq[AdditionalCoordinate]],
30+
getBucketPositions: (Long, Vec3Int) => Fox[Set[Vec3IntProto]],
31+
getDataForBucketPositions: (
32+
Seq[Vec3Int],
33+
Vec3Int,
34+
Option[Seq[AdditionalCoordinate]]) => Fox[(Seq[Box[Array[Byte]]], ElementClass.Value)])(
3035
implicit ec: ExecutionContext): Fox[Long] =
3136
for {
3237
bucketPositionsProtos: Set[Vec3IntProto] <- getBucketPositions(segmentId, mag)
3338
bucketPositionsInMag = bucketPositionsProtos.map(vec3IntFromProto)
34-
volumeBoxes <- Fox.serialSequence(bucketPositionsInMag.toList)(bucketPosition =>
35-
for {
36-
dataTyped: Array[SegmentInteger] <- getTypedDataForBucketPosition(bucketPosition, mag, additionalCoordinates)
37-
count = dataTyped.count(segmentInteger => segmentInteger.toLong == segmentId)
38-
} yield count.toLong)
39-
counts <- Fox.combined(volumeBoxes.map(_.toFox))
39+
(bucketBoxes, elementClass) <- getDataForBucketPositions(bucketPositionsInMag.toSeq, mag, additionalCoordinates)
40+
counts <- Fox.serialCombined(bucketBoxes.toList) {
41+
case Full(bucketBytes) =>
42+
tryo(
43+
bucketScanner.countSegmentVoxels(bucketBytes,
44+
ElementClass.bytesPerElement(elementClass),
45+
ElementClass.isSigned(elementClass),
46+
segmentId)).toFox
47+
case Empty => Full(0L).toFox
48+
case f: Failure => f.toFox
49+
}
4050
} yield counts.sum
4151

4252
// Returns the bounding box in voxels in the target mag
43-
def calculateSegmentBoundingBox(segmentId: Long,
44-
mag: Vec3Int,
45-
additionalCoordinates: Option[Seq[AdditionalCoordinate]],
46-
getBucketPositions: (Long, Vec3Int) => Fox[Set[Vec3IntProto]],
47-
getTypedDataForBucketPosition: (
48-
Vec3Int,
49-
Vec3Int,
50-
Option[Seq[AdditionalCoordinate]]) => Fox[Array[SegmentInteger]])(
53+
def calculateSegmentBoundingBox(
54+
segmentId: Long,
55+
mag: Vec3Int,
56+
additionalCoordinates: Option[Seq[AdditionalCoordinate]],
57+
getBucketPositions: (Long, Vec3Int) => Fox[Set[Vec3IntProto]],
58+
getDataForBucketPositions: (
59+
Seq[Vec3Int],
60+
Vec3Int,
61+
Option[Seq[AdditionalCoordinate]]) => Fox[(Seq[Box[Array[Byte]]], ElementClass.Value)])(
5162
implicit ec: ExecutionContext): Fox[BoundingBox] =
5263
for {
5364
allBucketPositions: Set[Vec3IntProto] <- getBucketPositions(segmentId, mag)
@@ -58,14 +69,16 @@ trait SegmentStatistics extends ProtoGeometryImplicits with FoxImplicits {
5869
Int.MinValue,
5970
Int.MinValue,
6071
Int.MinValue) //topleft, bottomright
61-
_ <- Fox.serialCombined(relevantBucketPositions.iterator)(
62-
bucketPosition =>
63-
extendBoundingBoxByData(mag,
64-
segmentId,
65-
boundingBoxMutable,
66-
bucketPosition,
67-
additionalCoordinates,
68-
getTypedDataForBucketPosition))
72+
(bucketBoxes, elementClass) <- getDataForBucketPositions(relevantBucketPositions.toSeq,
73+
mag,
74+
additionalCoordinates)
75+
_ <- Fox.serialCombined(relevantBucketPositions.zip(bucketBoxes)) {
76+
case (bucketPosition, Full(bucketData)) =>
77+
Fox.successful(
78+
extendBoundingBoxByData(segmentId, boundingBoxMutable, bucketPosition, bucketData, elementClass))
79+
case (_, Empty) => Fox.successful(())
80+
case (_, f: Failure) => f.toFox
81+
}
6982
} yield
7083
if (boundingBoxMutable.exists(item => item == Int.MaxValue || item == Int.MinValue)) {
7184
BoundingBox.empty
@@ -93,44 +106,30 @@ trait SegmentStatistics extends ProtoGeometryImplicits with FoxImplicits {
93106
.map(vec3IntFromProto)
94107
}
95108

96-
private def extendBoundingBoxByData(mag: Vec3Int,
97-
segmentId: Long,
98-
mutableBoundingBox: scala.collection.mutable.ListBuffer[Int],
109+
private def extendBoundingBoxByData(segmentId: Long,
110+
boundingBoxMutable: scala.collection.mutable.ListBuffer[Int],
99111
bucketPosition: Vec3Int,
100-
additionalCoordinates: Option[Seq[AdditionalCoordinate]],
101-
getTypedDataForBucketPosition: (
102-
Vec3Int,
103-
Vec3Int,
104-
Option[Seq[AdditionalCoordinate]]) => Fox[Array[SegmentInteger]]): Fox[Unit] =
105-
for {
106-
dataTyped: Array[SegmentInteger] <- getTypedDataForBucketPosition(bucketPosition, mag, additionalCoordinates)
107-
bucketTopLeftInTargetMagVoxels = bucketPosition * DataLayer.bucketLength
108-
_ = scanDataAndExtendBoundingBox(dataTyped, bucketTopLeftInTargetMagVoxels, segmentId, mutableBoundingBox)
109-
} yield ()
110-
111-
private def scanDataAndExtendBoundingBox(dataTyped: Array[SegmentInteger],
112-
bucketTopLeftInTargetMagVoxels: Vec3Int,
113-
segmentId: Long,
114-
mutableBoundingBox: scala.collection.mutable.ListBuffer[Int]): Unit =
115-
for {
116-
x <- 0 until DataLayer.bucketLength
117-
y <- 0 until DataLayer.bucketLength
118-
z <- 0 until DataLayer.bucketLength
119-
index = z * DataLayer.bucketLength * DataLayer.bucketLength + y * DataLayer.bucketLength + x
120-
} yield {
121-
if (dataTyped(index).toLong == segmentId) {
122-
val voxelPosition = bucketTopLeftInTargetMagVoxels + Vec3Int(x, y, z)
123-
extendBoundingBoxByPosition(mutableBoundingBox, voxelPosition)
124-
}
125-
}
126-
127-
private def extendBoundingBoxByPosition(mutableBoundingBox: scala.collection.mutable.ListBuffer[Int],
128-
position: Vec3Int): Unit = {
129-
mutableBoundingBox(0) = Math.min(mutableBoundingBox(0), position.x)
130-
mutableBoundingBox(1) = Math.min(mutableBoundingBox(1), position.y)
131-
mutableBoundingBox(2) = Math.min(mutableBoundingBox(2), position.z)
132-
mutableBoundingBox(3) = Math.max(mutableBoundingBox(3), position.x)
133-
mutableBoundingBox(4) = Math.max(mutableBoundingBox(4), position.y)
134-
mutableBoundingBox(5) = Math.max(mutableBoundingBox(5), position.z)
112+
bucketBytes: Array[Byte],
113+
elementClass: ElementClass.Value): Unit = {
114+
val bucketTopLeftInTargetMagVoxels = bucketPosition * DataLayer.bucketLength
115+
val extendedBBArray: Array[Int] =
116+
bucketScanner.extendSegmentBoundingBox(
117+
bucketBytes,
118+
ElementClass.bytesPerElement(elementClass),
119+
ElementClass.isSigned(elementClass),
120+
DataLayer.bucketLength,
121+
segmentId,
122+
bucketTopLeftInTargetMagVoxels.x,
123+
bucketTopLeftInTargetMagVoxels.y,
124+
bucketTopLeftInTargetMagVoxels.z,
125+
boundingBoxMutable(0),
126+
boundingBoxMutable(1),
127+
boundingBoxMutable(2),
128+
boundingBoxMutable(3),
129+
boundingBoxMutable(4),
130+
boundingBoxMutable(5)
131+
)
132+
(0 until 6).foreach(i => boundingBoxMutable(i) = extendedBBArray(i))
135133
}
134+
136135
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/DataServiceRequests.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ case class DataServiceDataRequest(
1919
dataLayer: DataLayer,
2020
cuboid: Cuboid,
2121
settings: DataServiceRequestSettings
22-
)
22+
) {
23+
def isSingleBucket: Boolean = cuboid.isSingleBucket(DataLayer.bucketLength)
24+
}
2325

2426
case class DataReadInstruction(
2527
baseDir: Path,

0 commit comments

Comments
 (0)