Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ import com.scalableminds.webknossos.datastore.helpers.{
import com.scalableminds.webknossos.datastore.models.datasource.inbox.InboxDataSource
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSource, DataSourceId, GenericDataSource}
import com.scalableminds.webknossos.datastore.services._
import com.scalableminds.webknossos.datastore.services.connectome.ConnectomeFileService
import com.scalableminds.webknossos.datastore.services.mesh.{MeshFileService, MeshMappingHelper}
import com.scalableminds.webknossos.datastore.services.segmentindex.SegmentIndexFileService
import com.scalableminds.webknossos.datastore.services.uploading._
import com.scalableminds.webknossos.datastore.storage.DataVaultService
import com.scalableminds.webknossos.datastore.services.connectome.{
ByAgglomerateIdsRequest,
BySynapseIdsRequest,
ConnectomeFileService,
SynapticPartnerDirection
}
import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
Expand Down Expand Up @@ -266,8 +266,8 @@ class DataSourceController @Inject()(
): Action[AnyContent] = Action.async { implicit request =>
accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataset(datasetId)) {
for {
(dataSource, dataLayer) <- datasetCache.getWithLayer(datasetId, dataLayerName) ~> NOT_FOUND
agglomerateList = agglomerateService.listAgglomeratesFiles(dataSource.id, dataLayer)
(_, dataLayer) <- datasetCache.getWithLayer(datasetId, dataLayerName) ~> NOT_FOUND
agglomerateList = agglomerateService.listAgglomeratesFiles(dataLayer)
} yield Ok(Json.toJson(agglomerateList))
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,18 @@ package com.scalableminds.webknossos.datastore.services.connectome

import com.scalableminds.util.accesscontext.TokenContext
import com.scalableminds.util.cache.AlfuCache
import com.scalableminds.util.io.PathUtils
import com.scalableminds.util.tools.Box.tryo
import com.scalableminds.util.tools.{Box, Fox, FoxImplicits}
import com.scalableminds.webknossos.datastore.DataStoreConfig
import com.scalableminds.util.tools.{Fox, FoxImplicits}
import com.scalableminds.webknossos.datastore.models.datasource.{
DataLayer,
DataSourceId,
LayerAttachment,
LayerAttachmentDataformat
}
import com.scalableminds.webknossos.datastore.services.connectome.SynapticPartnerDirection.SynapticPartnerDirection
import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
import com.typesafe.scalalogging.LazyLogging
import org.apache.commons.io.FilenameUtils
import play.api.i18n.{Messages, MessagesProvider}
import play.api.libs.json.{Json, OFormat}

import java.nio.file.Path
import javax.inject.Inject
import scala.collection.mutable.ListBuffer
import scala.concurrent.ExecutionContext
Expand Down Expand Up @@ -82,17 +76,11 @@ object ConnectomeFileNameWithMappingName {

case class ConnectomeFileKey(dataSourceId: DataSourceId, layerName: String, attachment: LayerAttachment)

class ConnectomeFileService @Inject()(config: DataStoreConfig,
remoteSourceDescriptorService: RemoteSourceDescriptorService,
hdf5ConnectomeFileService: Hdf5ConnectomeFileService,
class ConnectomeFileService @Inject()(hdf5ConnectomeFileService: Hdf5ConnectomeFileService,
zarrConnectomeFileService: ZarrConnectomeFileService)
extends FoxImplicits
with LazyLogging {

private val dataBaseDir = Path.of(config.Datastore.baseDirectory)
private val localConnectomesDir = "connectomes"
private val hdf5ConnectomeFileExtension = "hdf5"

private val connectomeFileKeyCache
: AlfuCache[(DataSourceId, String, String), ConnectomeFileKey] = AlfuCache() // dataSourceId, layerName, connectomeFileName → ConnectomeFileKey

Expand All @@ -104,60 +92,28 @@ class ConnectomeFileService @Inject()(config: DataStoreConfig,

private def lookUpConnectomeFileKeyImpl(dataSourceId: DataSourceId,
dataLayer: DataLayer,
connectomeFileName: String): Box[ConnectomeFileKey] = {
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
case Some(attachments) => attachments.connectomes.find(_.name == connectomeFileName)
case None => None
}
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
connectomeFileName: String): Option[ConnectomeFileKey] =
for {
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
attachment.copy(
path =
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
})
localFallbackAttachment = LayerAttachment(
connectomeFileName,
localDatasetDir
.resolve(dataLayer.name)
.resolve(localConnectomesDir)
.resolve(connectomeFileName + "." + hdf5ConnectomeFileExtension)
.toUri,
LayerAttachmentDataformat.hdf5
)
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
attachment <- dataLayer.attachments match {
case Some(attachments) => attachments.connectomes.find(_.name == connectomeFileName)
case None => None
}
} yield
ConnectomeFileKey(
dataSourceId,
dataLayer.name,
selectedAttachment
attachment
)
}

def listConnectomeFiles(dataSourceId: DataSourceId, dataLayer: DataLayer)(
implicit ec: ExecutionContext,
tc: TokenContext,
m: MessagesProvider): Fox[List[ConnectomeFileNameWithMappingName]] = {
val attachedConnectomeFileNames = dataLayer.attachments.map(_.connectomes).getOrElse(Seq.empty).map(_.name).toSet

val layerDir =
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
val scannedConnectomeFileNames = PathUtils
.listFiles(layerDir.resolve(localConnectomesDir),
silent = true,
PathUtils.fileExtensionFilter(hdf5ConnectomeFileExtension))
.map { paths =>
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
}
.toOption
.getOrElse(Nil)
.toSet

val allConnectomeFileNames = attachedConnectomeFileNames ++ scannedConnectomeFileNames
val connectomeFileNames = dataLayer.attachments.map(_.connectomes).getOrElse(Seq.empty).map(_.name)

Fox.fromFuture(
Fox
.serialSequence(allConnectomeFileNames.toSeq) { connectomeFileName =>
.serialSequence(connectomeFileNames) { connectomeFileName =>
for {
connectomeFileKey <- lookUpConnectomeFileKey(dataSourceId, dataLayer, connectomeFileName) ?~> Messages(
"connectome.file.lookup.failed",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,60 +3,29 @@ package com.scalableminds.webknossos.datastore.services.mapping
import com.scalableminds.util.accesscontext.TokenContext
import com.scalableminds.util.cache.AlfuCache
import com.scalableminds.util.geometry.Vec3Int
import com.scalableminds.util.io.PathUtils
import com.scalableminds.util.time.Instant
import com.scalableminds.util.tools.Box.tryo
import com.scalableminds.util.tools.{Box, Fox, FoxImplicits}
import com.scalableminds.util.tools.{Fox, FoxImplicits}
import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph
import com.scalableminds.webknossos.datastore.DataStoreConfig
import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
import com.scalableminds.webknossos.datastore.models.datasource.{
DataLayer,
DataSourceId,
LayerAttachment,
LayerAttachmentDataformat
}
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSourceId, LayerAttachmentDataformat}
import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, RemoteSourceDescriptorService}
import com.scalableminds.webknossos.datastore.storage.AgglomerateFileKey
import com.typesafe.scalalogging.LazyLogging
import org.apache.commons.io.FilenameUtils

import java.nio.file.Path
import javax.inject.Inject
import scala.concurrent.ExecutionContext
import scala.concurrent.duration.DurationInt

class AgglomerateService @Inject()(config: DataStoreConfig,
zarrAgglomerateService: ZarrAgglomerateService,
hdf5AgglomerateService: Hdf5AgglomerateService,
remoteSourceDescriptorService: RemoteSourceDescriptorService)
class AgglomerateService @Inject()(zarrAgglomerateService: ZarrAgglomerateService,
hdf5AgglomerateService: Hdf5AgglomerateService)
extends LazyLogging
with FoxImplicits {
private val localAgglomeratesDir = "agglomerates"
private val hdf5AgglomerateFileExtension = "hdf5"
private val dataBaseDir = Path.of(config.Datastore.baseDirectory)

private val agglomerateFileKeyCache
: AlfuCache[(DataSourceId, String, String), AgglomerateFileKey] = AlfuCache() // dataSourceId, layerName, mappingName → AgglomerateFileKey

def listAgglomeratesFiles(dataSourceId: DataSourceId, dataLayer: DataLayer): Set[String] = {
val attachedAgglomerateFileNames = dataLayer.attachments.map(_.agglomerates).getOrElse(Seq.empty).map(_.name).toSet

val layerDir =
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
val scannedAgglomerateFileNames = PathUtils
.listFiles(layerDir.resolve(localAgglomeratesDir),
silent = true,
PathUtils.fileExtensionFilter(hdf5AgglomerateFileExtension))
.map { paths =>
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
}
.toOption
.getOrElse(Nil)
.toSet

attachedAgglomerateFileNames ++ scannedAgglomerateFileNames
}
def listAgglomeratesFiles(dataLayer: DataLayer): Seq[String] =
dataLayer.attachments.map(_.agglomerates).getOrElse(Seq.empty).map(_.name)

def clearCaches(dataSourceId: DataSourceId, layerNameOpt: Option[String]): Int = {
agglomerateFileKeyCache.clear {
Expand All @@ -83,35 +52,18 @@ class AgglomerateService @Inject()(config: DataStoreConfig,

private def lookUpAgglomerateFileImpl(dataSourceId: DataSourceId,
dataLayer: DataLayer,
mappingName: String): Box[AgglomerateFileKey] = {
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
case Some(attachments) => attachments.agglomerates.find(_.name == mappingName)
case None => None
}
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
mappingName: String): Option[AgglomerateFileKey] =
for {
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
attachment.copy(
path =
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
})
localFallbackAttachment = LayerAttachment(
mappingName,
localDatasetDir
.resolve(dataLayer.name)
.resolve(localAgglomeratesDir)
.resolve(mappingName + "." + hdf5AgglomerateFileExtension)
.toUri,
LayerAttachmentDataformat.hdf5
)
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
attachment <- dataLayer.attachments match {
case Some(attachments) => attachments.agglomerates.find(_.name == mappingName)
case None => None
}
} yield
AgglomerateFileKey(
dataSourceId,
dataLayer.name,
selectedAttachment
attachment
)
}

def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
tc: TokenContext): Fox[Array[Byte]] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,16 @@ package com.scalableminds.webknossos.datastore.services.mesh

import com.scalableminds.util.accesscontext.TokenContext
import com.scalableminds.util.cache.AlfuCache
import com.scalableminds.util.io.PathUtils
import com.scalableminds.util.tools.{Fox, FoxImplicits}
import com.scalableminds.webknossos.datastore.DataStoreConfig
import com.scalableminds.webknossos.datastore.models.datasource.{
DataLayer,
DataSourceId,
LayerAttachment,
LayerAttachmentDataformat
}
import com.scalableminds.webknossos.datastore.services.ArrayArtifactHashing
import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
import com.scalableminds.util.tools.Box.tryo
import com.scalableminds.util.tools.Box
import org.apache.commons.io.FilenameUtils
import play.api.i18n.{Messages, MessagesProvider}
import play.api.libs.json.{Json, OFormat}

import java.nio.file.Path
import javax.inject.Inject
import scala.concurrent.ExecutionContext

Expand Down Expand Up @@ -64,17 +56,10 @@ object MeshFileInfo {
implicit val jsonFormat: OFormat[MeshFileInfo] = Json.format[MeshFileInfo]
}

class MeshFileService @Inject()(config: DataStoreConfig,
hdf5MeshFileService: Hdf5MeshFileService,
class MeshFileService @Inject()(hdf5MeshFileService: Hdf5MeshFileService,
zarrMeshFileService: ZarrMeshFileService,
neuroglancerPrecomputedMeshService: NeuroglancerPrecomputedMeshFileService,
remoteSourceDescriptorService: RemoteSourceDescriptorService)
extends FoxImplicits
with ArrayArtifactHashing {

private val dataBaseDir = Path.of(config.Datastore.baseDirectory)
private val localMeshesDir = "meshes"
private val hdf5MeshFileExtension = "hdf5"
neuroglancerPrecomputedMeshService: NeuroglancerPrecomputedMeshFileService)
extends FoxImplicits {

private val meshFileKeyCache
: AlfuCache[(DataSourceId, String, String), MeshFileKey] = AlfuCache() // dataSourceId, layerName, meshFileName → MeshFileKey
Expand All @@ -86,57 +71,27 @@ class MeshFileService @Inject()(config: DataStoreConfig,

private def lookUpMeshFileKeyImpl(dataSourceId: DataSourceId,
dataLayer: DataLayer,
meshFileName: String): Box[MeshFileKey] = {
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
case Some(attachments) => attachments.meshes.find(_.name == meshFileName)
case None => None
}
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
meshFileName: String): Option[MeshFileKey] =
for {
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
attachment.copy(
path =
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
})
localFallbackAttachment = LayerAttachment(
meshFileName,
localDatasetDir
.resolve(dataLayer.name)
.resolve(localMeshesDir)
.resolve(meshFileName + "." + hdf5MeshFileExtension)
.toUri,
LayerAttachmentDataformat.hdf5
)
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
registeredAttachment <- dataLayer.attachments match {
case Some(attachments) => attachments.meshes.find(_.name == meshFileName)
case None => None
}
} yield
MeshFileKey(
dataSourceId,
dataLayer.name,
selectedAttachment
registeredAttachment
)
}

def listMeshFiles(dataSourceId: DataSourceId, dataLayer: DataLayer)(implicit ec: ExecutionContext,
tc: TokenContext,
m: MessagesProvider): Fox[Seq[MeshFileInfo]] = {
val attachedMeshFileNames = dataLayer.attachments.map(_.meshes).getOrElse(Seq.empty).map(_.name).toSet

val layerDir =
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
val scannedMeshFileNames = PathUtils
.listFiles(layerDir.resolve(localMeshesDir), silent = true, PathUtils.fileExtensionFilter(hdf5MeshFileExtension))
.map { paths =>
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
}
.toOption
.getOrElse(Nil)
.toSet

val allMeshFileNames = attachedMeshFileNames ++ scannedMeshFileNames
val meshFileNames = dataLayer.attachments.map(_.meshes).getOrElse(Seq.empty).map(_.name)

Fox.fromFuture(
Fox
.serialSequence(allMeshFileNames.toSeq) { meshFileName =>
.serialSequence(meshFileNames) { meshFileName =>
for {
meshFileKey <- lookUpMeshFileKey(dataSourceId, dataLayer, meshFileName) ?~> Messages(
"mesh.file.lookup.failed",
Expand Down
Loading