Store dataset properties in DB

MIGRATIONS.unreleased.md

@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 - [131-more-indices-on-users.sql](conf/evolutions/131-more-indices-on-users.sql)
 - [132-remove-stored-meshes.sql](conf/evolutions/132-remove-stored-meshes.sql)
+- [133-datasource-properties-in-db.sql](conf/evolutions/133-datasource-properties-in-db.sql)

app/models/dataset/Dataset.scala

@@ -750,12 +750,31 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
   def updateMags(datasetId: ObjectId, dataLayersOpt: Option[List[DataLayer]]): Fox[Unit] = {
     val clearQuery = q"DELETE FROM webknossos.dataset_mags WHERE _dataset = $datasetId".asUpdate
     val insertQueries = dataLayersOpt.getOrElse(List.empty).flatMap { layer: DataLayer =>
-      layer.resolutions.distinct.map { mag: Vec3Int =>
-        {
-          q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag)
-                VALUES($datasetId, ${layer.name}, $mag)""".asUpdate
-        }
+      layer.magsOpt match {
+        case Some(mags) =>
+          mags.map(mag => {
+            q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, axisOrder, channelIndex, credentialId)
+                VALUES($datasetId, ${layer.name}, ${mag.mag}, ${mag.axisOrder
+              .map(Json.toJson(_))}, ${mag.channelIndex}, ${mag.credentialId})
+           """.asUpdate
+          })
+        case None =>
+          layer.wkwResolutionsOpt match {
+            case Some(resolutions) =>
+              resolutions.map(wkwResolution => {
+                q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, cubeLength)
+                 VALUES ($datasetId, ${layer.name}, ${wkwResolution.resolution}, ${wkwResolution.cubeLength})""".asUpdate
+              })
+            case None =>
+              layer.resolutions.distinct.map { mag: Vec3Int =>
+                {
+                  q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag)
+                    VALUES($datasetId, ${layer.name}, $mag)""".asUpdate
+                }
+              }
+          }
       }
+
     }
     replaceSequentiallyAsTransaction(clearQuery, insertQueries)
   }
@@ -897,31 +916,39 @@ class DatasetLayerDAO @Inject()(
     layer match {
       case s: AbstractSegmentationLayer =>
         val mappings = s.mappings.getOrElse(Set()).toList
-        q"""INSERT INTO webknossos.dataset_layers(_dataset, name, category, elementClass, boundingBox, largestSegmentId, mappings, defaultViewConfiguration, adminViewConfiguration)
+        q"""INSERT INTO webknossos.dataset_layers(_dataset, name, category, elementClass, boundingBox, largestSegmentId, mappings, defaultViewConfiguration, adminViewConfiguration, dataFormat, numChannels)
                     VALUES($datasetId, ${s.name}, ${s.category}, ${s.elementClass},
                     ${s.boundingBox}, ${s.largestSegmentId}, $mappings,
                     ${s.defaultViewConfiguration.map(Json.toJson(_))},
-                    ${s.adminViewConfiguration.map(Json.toJson(_))})
+                    ${s.adminViewConfiguration.map(Json.toJson(_))},
+                    ${s.dataFormat}, ${s.numChannels})
             ON CONFLICT (_dataset, name) DO UPDATE
             SET
               category = ${s.category},
               elementClass = ${s.elementClass},
               boundingBox = ${s.boundingBox},
               largestSegmentId = ${s.largestSegmentId},
               mappings = $mappings,
-              defaultViewConfiguration = ${s.defaultViewConfiguration.map(Json.toJson(_))}""".asUpdate
+              defaultViewConfiguration = ${s.defaultViewConfiguration.map(Json.toJson(_))},
+              adminViewConfiguration = ${s.adminViewConfiguration.map(Json.toJson(_))},
+              numChannels = ${s.numChannels},
+              dataFormat = ${s.dataFormat} """.asUpdate
       case d: AbstractDataLayer =>
-        q"""INSERT INTO webknossos.dataset_layers(_dataset, name, category, elementClass, boundingBox, defaultViewConfiguration, adminViewConfiguration)
+        q"""INSERT INTO webknossos.dataset_layers(_dataset, name, category, elementClass, boundingBox, defaultViewConfiguration, adminViewConfiguration, dataFormat, numChannels)
                     VALUES($datasetId, ${d.name}, ${d.category}, ${d.elementClass},
                     ${d.boundingBox},
                     ${d.defaultViewConfiguration.map(Json.toJson(_))},
-                    ${d.adminViewConfiguration.map(Json.toJson(_))})
+                    ${d.adminViewConfiguration.map(Json.toJson(_))},
+                    ${d.dataFormat}, ${d.numChannels})
             ON CONFLICT (_dataset, name) DO UPDATE
             SET
               category = ${d.category},
               elementClass = ${d.elementClass},
               boundingBox = ${d.boundingBox},
-              defaultViewConfiguration = ${d.defaultViewConfiguration.map(Json.toJson(_))}""".asUpdate
+              defaultViewConfiguration = ${d.defaultViewConfiguration.map(Json.toJson(_))},
+              adminViewConfiguration = ${d.adminViewConfiguration.map(Json.toJson(_))},
+              numChannels = ${d.numChannels},
+              dataFormat = ${d.dataFormat}""".asUpdate
       case _ => throw new Exception("DataLayer type mismatch")
     }
 

conf/evolutions/133-datasource-properties-in-db.sql

@@ -0,0 +1,23 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 132, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+-- The aim of this migration is to have all properties of datasources that are saved in the datasource-properties.json
+-- file saved in the database.
+
+CREATE TYPE webknossos.DATASET_LAYER_DATAFORMAT AS ENUM ('wkw','zarr','zarr3','n5','neuroglancerPrecomputed');
+
+ALTER TABLE webknossos.dataset_layers
+  ADD COLUMN IF NOT EXISTS numChannels INT,
+  ADD COLUMN IF NOT EXISTS dataFormat webknossos.DATASET_LAYER_DATAFORMAT;
+
+ALTER TABLE webknossos.dataset_mags
+  ADD COLUMN IF NOT EXISTS credentialId TEXT,
+  ADD COLUMN IF NOT EXISTS axisOrder JSONB CONSTRAINT axisOrder_requiredKeys CHECK (axisOrder ? 'x' AND axisOrder ? 'y'),
+  ADD COLUMN IF NOT EXISTS channelIndex INT,
+  ADD COLUMN IF NOT EXISTS cubeLength INT;
+  -- legacy credentials omitted
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 133;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/133-datasource-properties-in-db.sql

@@ -0,0 +1,19 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 133, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE webknossos.dataset_layers
+  DROP COLUMN IF EXISTS numChannels,
+  DROP COLUMN IF EXISTS dataFormat;
+
+DROP TYPE IF EXISTS webknossos.DATASET_LAYER_DATAFORMAT;
+
+ALTER TABLE webknossos.dataset_mags
+  DROP COLUMN IF EXISTS credentialId,
+  DROP COLUMN IF EXISTS axisOrder,
+  DROP COLUMN IF EXISTS channelIndex,
+  DROP COLUMN IF EXISTS cubeLength;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 132;
+
+COMMIT TRANSACTION;

conf/webknossos.latest.routes

@@ -17,7 +17,6 @@ POST          /maintenances
 GET           /maintenances/:id                                                                 controllers.MaintenanceController.readOne(id: ObjectId)
 PUT           /maintenances/:id                                                                 controllers.MaintenanceController.update(id: ObjectId)
 DELETE        /maintenances/:id                                                                 controllers.MaintenanceController.delete(id: ObjectId)
-POST          /maintenances                                                                     controllers.MaintenanceController.create()
 POST          /maintenances/adHoc                                                               controllers.MaintenanceController.createAdHocMaintenance()
 GET           /maintenance/revokeExpiredCredits                                                 controllers.CreditTransactionController.revokeExpiredCredits()
 

test/db/dataSet_layers.csv

@@ -1,6 +1,6 @@
-_dataSet,name,category,elementClass,boundingBox,largestSegmentId,mappings,defaultViewConfiguration,adminViewConfiguration
-'59e9cfbdba632ac2ab8b23b3','color_1','color','uint8','(0,0,0,512,512,256)',,,,
-'59e9cfbdba632ac2ab8b23b3','color_2','color','uint8','(0,0,0,512,512,256)',,,,
-'59e9cfbdba632ac2ab8b23b3','color_3','color','uint8','(0,0,0,512,512,256)',,,,
-'59e9cfbdba632ac2ab8b23b5','color','color','uint8','(3072,3072,512,1024,1024,1024)',,,,
-'59e9cfbdba632ac2ab8b23b5','segmentation','segmentation','uint32','(3072,3072,512,1024,1024,1024)',2504697,'{}',,
+_dataSet,name,category,elementClass,boundingBox,largestSegmentId,mappings,defaultViewConfiguration,adminViewConfiguration,numChannels,dataFormat
+'59e9cfbdba632ac2ab8b23b3','color_1','color','uint8','(0,0,0,512,512,256)',,,,,,wkw
+'59e9cfbdba632ac2ab8b23b3','color_2','color','uint8','(0,0,0,512,512,256)',,,,,,wkw
+'59e9cfbdba632ac2ab8b23b3','color_3','color','uint8','(0,0,0,512,512,256)',,,,,,wkw
+'59e9cfbdba632ac2ab8b23b5','color','color','uint8','(3072,3072,512,1024,1024,1024)',,,,,,wkw
+'59e9cfbdba632ac2ab8b23b5','segmentation','segmentation','uint32','(3072,3072,512,1024,1024,1024)',2504697,'{}',,,,wkw

test/db/dataSet_mags.csv

@@ -1,26 +1,26 @@
-_dataSet,dataLayerName,mag,scale,path,realPath,hasLocalData
-'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)','uninitialized','uninitialized',false
-'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)','uninitialized','uninitialized',false
+_dataSet,dataLayerName,mag,scale,path,realPath,hasLocalData,channelIndex,cubeLength,credentialId
+'59e9cfbdba632ac2ab8b23b3','color_1','(1,1,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_1','(2,2,2)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_1','(4,4,4)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_1','(8,8,8)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_1','(16,16,16)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_2','(1,1,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_2','(2,2,2)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_2','(4,4,4)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_2','(8,8,8)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_2','(16,16,16)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_3','(1,1,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_3','(2,2,2)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_3','(4,4,4)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_3','(8,8,8)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b3','color_3','(16,16,16)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','color','(1,1,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','color','(2,2,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','color','(4,4,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','color','(8,8,2)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','color','(16,16,4)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','segmentation','(1,1,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','segmentation','(2,2,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','segmentation','(4,4,1)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','segmentation','(8,8,2)','uninitialized','uninitialized',false,,,,
+'59e9cfbdba632ac2ab8b23b5','segmentation','(16,16,4)','uninitialized','uninitialized',false,,,,

tools/postgres/schema.sql

@@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(132);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(133);
 COMMIT TRANSACTION;
 
 
@@ -126,6 +126,7 @@ CREATE TABLE webknossos.datasets(
 
 CREATE TYPE webknossos.DATASET_LAYER_CATEGORY AS ENUM ('color', 'mask', 'segmentation');
 CREATE TYPE webknossos.DATASET_LAYER_ELEMENT_CLASS AS ENUM ('uint8', 'uint16', 'uint24', 'uint32', 'uint64', 'float', 'double', 'int8', 'int16', 'int32', 'int64');
+CREATE TYPE webknossos.DATASET_LAYER_DATAFORMAT AS ENUM ('wkw','zarr','zarr3','n5','neuroglancerPrecomputed');
 CREATE TABLE webknossos.dataset_layers(
   _dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
   name TEXT NOT NULL,
@@ -136,6 +137,8 @@ CREATE TABLE webknossos.dataset_layers(
   mappings TEXT[],
   defaultViewConfiguration JSONB,
   adminViewConfiguration JSONB,
+  numChannels INT,
+  dataFormat webknossos.DATASET_LAYER_DATAFORMAT,
   PRIMARY KEY(_dataset, name),
   CONSTRAINT defaultViewConfigurationIsJsonObject CHECK(jsonb_typeof(defaultViewConfiguration) = 'object'),
   CONSTRAINT adminViewConfigurationIsJsonObject CHECK(jsonb_typeof(adminViewConfiguration) = 'object')
@@ -172,6 +175,10 @@ CREATE TABLE webknossos.dataset_mags(
   path TEXT,
   realPath TEXT,
   hasLocalData BOOLEAN NOT NULL DEFAULT FALSE,
+  axisOrder JSONB CONSTRAINT axisOrder_requiredKeys CHECK (axisOrder ? 'x' AND axisOrder ? 'y'),
+  channelIndex INT,
+  cubeLength INT,
+  credentialId TEXT,
   PRIMARY KEY (_dataset, dataLayerName, mag)
 );
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/AxisOrder.scala

@@ -20,7 +20,6 @@ case class AxisOrder(x: Int, y: Int, z: Option[Int], c: Option[Int] = None) {
     val lengthOfC = if (c.isDefined) 1 else 0
     lengthOfC + 2 + lengthOfZ
   }
-
 }
 
 object AxisOrder {