Resolve symbolic links to mags and store them in db

MIGRATIONS.unreleased.md

@@ -18,3 +18,5 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
     - Example command for the migration: `PG_PASSWORD=myPassword python main.py --src localhost:7500 --dst localhost:7155 --num_threads 20 --postgres webknossos@localhost:5430/webknossos`
 
 ### Postgres Evolutions:
+
+- [126-mag-real-paths.sql](conf/evolutions/126-mag-real-paths.sql)

app/controllers/DatasetController.scala

@@ -71,6 +71,12 @@ object SegmentAnythingMaskParameters {
   implicit val jsonFormat: Format[SegmentAnythingMaskParameters] = Json.format[SegmentAnythingMaskParameters]
 }
 
+case class MagLinkInfo(mag: DatasetMagInfo, linkedMags: Seq[DatasetMagInfo])
+
+object MagLinkInfo {
+  implicit val jsonFormat: Format[MagLinkInfo] = Json.format[MagLinkInfo]
+}
+
 class DatasetController @Inject()(userService: UserService,
                                   userDAO: UserDAO,
                                   datasetService: DatasetService,
@@ -126,6 +132,18 @@ class DatasetController @Inject()(userService: UserService,
       }
     }
 
+  def getLinkedMags(datasetId: ObjectId, dataLayerName: String): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        _ <- datasetDAO.findOne(datasetId) ?~> notFoundMessage(datasetId) ~> NOT_FOUND
+        _ <- Fox.bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN
+        magsAndLinkedMags <- datasetService.getPathsForDatalayer(datasetId, dataLayerName)
+        returnValues = magsAndLinkedMags.map {
+          case (mag, linkedMags) => MagLinkInfo(mag, linkedMags)
+        }
+      } yield Ok(Json.toJson(returnValues))
+    }
+
   def exploreRemoteDataset(): Action[List[WKExploreRemoteLayerParameters]] =
     sil.SecuredAction.async(validateJson[List[WKExploreRemoteLayerParameters]]) { implicit request =>
       for {

app/controllers/WKRemoteDataStoreController.scala

@@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.controllers.JobExportProperties
 import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
-import com.scalableminds.webknossos.datastore.services.DataStoreStatus
+import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
 import com.scalableminds.webknossos.datastore.services.uploading.{
   LinkedLayerIdentifier,
   ReserveAdditionalInformation,
@@ -221,6 +221,22 @@ class WKRemoteDataStoreController @Inject()(
     }
   }
 
+  def updatePaths(name: String, key: String): Action[JsValue] = Action.async(parse.json) { implicit request =>
+    dataStoreService.validateAccess(name, key) { _ =>
+      request.body.validate[List[DataSourcePathInfo]] match {
+        case JsSuccess(infos, _) =>
+          for {
+            _ <- datasetService.updateRealPaths(infos)(GlobalAccessContext)
+          } yield {
+            JsonOk
+          }
+        case e: JsError =>
+          logger.warn("Data store reported invalid json for data source paths.")
+          Fox.successful(JsonBadRequest(JsError.toJson(e)))
+      }
+    }
+  }
+
   def deleteDataset(name: String, key: String): Action[JsValue] = Action.async(parse.json) { implicit request =>
     dataStoreService.validateAccess(name, key) { _ =>
       for {

app/models/dataset/Dataset.scala

@@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
   ThinPlateSplineCorrespondences,
   DataLayerLike => DataLayer
 }
+import com.scalableminds.webknossos.datastore.services.MagPathInfo
 import com.scalableminds.webknossos.schema.Tables._
 import controllers.DatasetUpdateParameters
 
@@ -29,6 +30,7 @@ import models.organization.OrganizationDAO
 import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.json._
 import play.utils.UriEncoding
+import slick.dbio.DBIO
 import slick.jdbc.PostgresProfile.api._
 import slick.jdbc.TransactionIsolation.Serializable
 import slick.lifted.Rep
@@ -707,18 +709,45 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA
   }
 }
 
-class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) extends SimpleSQLDAO(sqlClient) {
-  private def parseRow(row: DatasetMagsRow): Fox[Vec3Int] =
+case class DatasetMagInfo(datasetId: ObjectId,
+                          dataLayerName: String,
+                          mag: Vec3Int,
+                          path: String,
+                          realPath: String,
+                          hasLocalData: Boolean)
+
+object DatasetMagInfo {
+  implicit val jsonFormat: Format[DatasetMagInfo] = Json.format[DatasetMagInfo]
+}
+class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
+    extends SQLDAO[MagPathInfo, DatasetMagsRow, DatasetMags](sqlClient) {
+  protected val collection = DatasetMags
+
+  protected def idColumn(x: DatasetMags): Rep[String] = x._Dataset
+
+  protected def isDeletedColumn(x: DatasetMags): Rep[Boolean] = false
+
+  protected def parse(row: DatasetMagsRow): Fox[MagPathInfo] =
     for {
       mag <- Vec3Int.fromList(parseArrayLiteral(row.mag).map(_.toInt)) ?~> "could not parse mag"
+    } yield
+      MagPathInfo(row.datalayername,
+                  mag,
+                  row.path.getOrElse("uninitialized"),
+                  row.realpath.getOrElse("uninitialized"),
+                  hasLocalData = row.haslocaldata)
+
+  private def parseMag(magArrayLiteral: String): Fox[Vec3Int] =
+    for {
+      mag <- Vec3Int.fromList(parseArrayLiteral(magArrayLiteral).map(_.toInt)) ?~> "could not parse mag"
     } yield mag
 
   def findMagForLayer(datasetId: ObjectId, dataLayerName: String): Fox[List[Vec3Int]] =
     for {
       rows <- run(DatasetMags.filter(r => r._Dataset === datasetId.id && r.datalayername === dataLayerName).result)
         .map(_.toList)
-      rowsParsed <- Fox.combined(rows.map(parseRow)) ?~> "could not parse mag row"
-    } yield rowsParsed
+      mags <- Fox.combined(rows.map(r => parseMag(r.mag))) ?~> "could not parse mag row"
+    } yield mags
 
   def updateMags(datasetId: ObjectId, dataLayersOpt: Option[List[DataLayer]]): Fox[Unit] = {
     val clearQuery = q"DELETE FROM webknossos.dataset_mags WHERE _dataset = $datasetId".asUpdate
@@ -733,6 +762,53 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
     replaceSequentiallyAsTransaction(clearQuery, insertQueries)
   }
 
+  def updateMagPathsForDataset(datasetId: ObjectId, magPaths: List[MagPathInfo]): Fox[Unit] =
+    for {
+      _ <- Fox.successful(())
+      updateQueries = magPaths.map(pathInfo => {
+        val magLiteral = s"(${pathInfo.mag.x}, ${pathInfo.mag.y}, ${pathInfo.mag.z})"
+        q"""UPDATE webknossos.dataset_mags
+                 SET path = ${pathInfo.path}, realPath = ${pathInfo.realPath}, haslocaldata = ${pathInfo.hasLocalData}
+                 WHERE _dataset = $datasetId
+                  AND datalayername = ${pathInfo.layerName}
+                  AND mag = CAST($magLiteral AS webknossos.vector3)""".asUpdate
+      })
+      composedQuery = DBIO.sequence(updateQueries)
+      _ <- run(
+        composedQuery.transactionally.withTransactionIsolation(Serializable),
+        retryCount = 50,
+        retryIfErrorContains = List(transactionSerializationError)
+      )
+    } yield ()
+
+  def findPathsForDatasetAndDatalayer(datasetId: ObjectId, dataLayerName: String): Fox[List[DatasetMagInfo]] =
+    for {
+      rows <- run(q"""SELECT $columns
+            FROM webknossos.dataset_mags
+            WHERE _dataset = $datasetId
+            AND datalayername = $dataLayerName""".as[DatasetMagsRow])
+      magInfos <- Fox.combined(rows.toList.map(parse))
+      datasetMagInfos = magInfos.map(magInfo =>
+        DatasetMagInfo(datasetId, magInfo.layerName, magInfo.mag, magInfo.path, magInfo.realPath, magInfo.hasLocalData))
+    } yield datasetMagInfos
+
+  def findAllByRealPath(realPath: String): Fox[List[DatasetMagInfo]] =
+    for {
+      rows <- run(q"""SELECT $columns
+            FROM webknossos.dataset_mags
+            WHERE realPath = $realPath""".as[DatasetMagsRow])
+      mags <- Fox.serialCombined(rows.toList)(r => parseMag(r.mag))
+      magInfos = rows.toList.zip(mags).map {
+        case (row, mag) =>
+          DatasetMagInfo(ObjectId(row._Dataset),
+                         row.datalayername,
+                         mag,
+                         row.path.getOrElse("uninitialized"),
+                         row.realpath.getOrElse("uninitialized"),
+                         row.haslocaldata)
+      }
+    } yield magInfos
+
 }
 
 class DatasetLayerDAO @Inject()(

app/models/dataset/DatasetService.scala

@@ -14,12 +14,13 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
   DataLayerLike => DataLayer
 }
 import com.scalableminds.webknossos.datastore.rpc.RPC
+import com.scalableminds.webknossos.datastore.services.DataSourcePathInfo
 import com.typesafe.scalalogging.LazyLogging
 import models.folder.FolderDAO
 import models.organization.{Organization, OrganizationDAO}
 import models.team._
 import models.user.{User, UserService}
-import net.liftweb.common.{Box, Full}
+import net.liftweb.common.{Box, Empty, EmptyBox, Full}
 import play.api.libs.json.{JsObject, Json}
 import security.RandomIDGenerator
 import utils.WkConf
@@ -33,6 +34,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                                dataStoreDAO: DataStoreDAO,
                                datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO,
                                datasetDataLayerDAO: DatasetLayerDAO,
+                               datasetMagsDAO: DatasetMagsDAO,
                                teamDAO: TeamDAO,
                                folderDAO: FolderDAO,
                                dataStoreService: DataStoreService,
@@ -338,6 +340,35 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       _ <- datasetDAO.updateUploader(dataset._id, Some(_uploader)) ?~> "dataset.uploader.forbidden"
     } yield ()
 
+  private def updateRealPath(pathInfo: DataSourcePathInfo)(implicit ctx: DBAccessContext): Fox[Unit] =
+    if (pathInfo.magPathInfos.isEmpty) {
+      Fox.successful(())
+    } else {
+      val dataset = datasetDAO.findOneByDataSourceId(pathInfo.dataSourceId).futureBox
+      dataset.flatMap {
+        case Full(dataset) => datasetMagsDAO.updateMagPathsForDataset(dataset._id, pathInfo.magPathInfos)
+        case Empty => // Dataset reported but ignored (non-existing/forbidden org)
+          Fox.successful(())
+        case e: EmptyBox =>
+          Fox.failure("dataset.notFound", e)
+      }
+    }
+
+  def updateRealPaths(pathInfos: List[DataSourcePathInfo])(implicit ctx: DBAccessContext): Fox[Unit] =
+    for {
+      _ <- Fox.serialCombined(pathInfos)(updateRealPath)
+    } yield ()
+
+  def getPathsForDatalayer(datasetId: ObjectId, layerName: String): Fox[List[(DatasetMagInfo, List[DatasetMagInfo])]] =
+    for {
+      magInfos <- datasetMagsDAO.findPathsForDatasetAndDatalayer(datasetId, layerName)
+      magInfosAndLinkedMags <- Fox.serialCombined(magInfos)(magInfo =>
+        for {
+          pathInfos <- datasetMagsDAO.findAllByRealPath(magInfo.realPath)
+        } yield (magInfo, pathInfos.filter(!_.equals(magInfo))))
+
+    } yield magInfosAndLinkedMags
+
   def publicWrites(dataset: Dataset,
                    requestingUserOpt: Option[User],
                    organization: Option[Organization] = None,

conf/evolutions/126-mag-real-paths.sql

@@ -0,0 +1,12 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 125, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE webknossos.dataset_mags ADD COLUMN realPath TEXT;
+ALTER TABLE webknossos.dataset_mags ADD COLUMN path TEXT;
+ALTER TABLE webknossos.dataset_mags ADD COLUMN hasLocalData BOOLEAN NOT NULL DEFAULT false;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 126;
+
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/126-mag-real-paths.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 126, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE webknossos.dataset_mags DROP COLUMN realPath;
+ALTER TABLE webknossos.dataset_mags DROP COLUMN path;
+ALTER TABLE webknossos.dataset_mags DROP COLUMN hasLocalData;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 125;
+
+COMMIT TRANSACTION;

conf/webknossos.latest.routes

@@ -91,6 +91,7 @@ DELETE        /datasets/:datasetId/sharingToken
 PATCH         /datasets/:datasetId/teams                                                        controllers.DatasetController.updateTeams(datasetId: ObjectId)
 GET           /datasets/:datasetId/layers/:layer/thumbnail                                      controllers.DatasetController.thumbnail(datasetId: ObjectId, layer: String, w: Option[Int], h: Option[Int], mappingName: Option[String], sharingToken: Option[String])
 POST          /datasets/:datasetId/layers/:layer/segmentAnythingMask                            controllers.DatasetController.segmentAnythingMask(datasetId: ObjectId, layer: String, intensityMin: Option[Float], intensityMax: Option[Float])
+GET           /datasets/:datasetId/layers/:layer/linkedMags                                     controllers.DatasetController.getLinkedMags(datasetId: ObjectId, layer: String)
 PUT           /datasets/:datasetId/clearThumbnailCache                                          controllers.DatasetController.removeFromThumbnailCache(datasetId: ObjectId)
 GET           /datasets/:datasetName/isValidNewName                                             controllers.DatasetController.isValidNewName(datasetName: String)
 GET           /datasets/:datasetId                                                              controllers.DatasetController.read(datasetId: ObjectId, sharingToken: Option[String])
@@ -108,6 +109,7 @@ DELETE        /folders/:id
 GET           /datastores                                                                       controllers.DataStoreController.list()
 PUT           /datastores/:name/datasource                                                      controllers.WKRemoteDataStoreController.updateOne(name: String, key: String)
 PUT           /datastores/:name/datasources                                                     controllers.WKRemoteDataStoreController.updateAll(name: String, key: String)
+PUT           /datastores/:name/datasources/paths                                               controllers.WKRemoteDataStoreController.updatePaths(name: String, key: String)
 PATCH         /datastores/:name/status                                                          controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
 POST          /datastores/:name/reserveUpload                                                   controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String)
 GET           /datastores/:name/getUnfinishedUploadsForUser                                     controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String)

test/db/dataSet_mags.csv

@@ -1,26 +1,26 @@
-_dataSet,dataLayerName,mag,scale
-'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)'
-'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)'
-'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)'
-'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)'
-'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)'
-'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)'
-'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)'
-'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)'
-'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)'
-'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)'
-'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)'
-'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)'
-'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)'
-'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)'
-'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)'
-'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)'
-'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)'
-'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)'
-'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)'
-'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)'
-'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)'
-'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)'
-'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)'
-'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)'
-'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)'
+_dataSet,dataLayerName,mag,scale,path,realPath,hasLocalData
+'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)','uninitialized','uninitialized',false
+'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)','uninitialized','uninitialized',false

tools/postgres/schema.sql

@@ -20,7 +20,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(125);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(126);
 COMMIT TRANSACTION;
 
 
@@ -178,6 +178,9 @@ CREATE TABLE webknossos.dataset_mags(
   _dataset CHAR(24) NOT NULL,
   dataLayerName VARCHAR(256),
   mag webknossos.VECTOR3 NOT NULL,
+  path TEXT,
+  realPath TEXT,
+  hasLocalData BOOLEAN NOT NULL DEFAULT false,
   PRIMARY KEY (_dataset, dataLayerName, mag)
 );