Route to register aiModel (#8127)

* Route to register aiModel

* update schema with new model categories. skip id taken check for the moment

* bump schema version

* add uniqueness checks for name + id. add sql evolution for enum values

* no transaction block in evolution

* delete incompatible models in reversion

Author: fm3

MIGRATIONS.unreleased.md

@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 - [121-worker-name.sql](conf/evolutions/121-worker-name.sql)
 - [122-resolution-to-mag.sql](conf/evolutions/122-resolution-to-mag.sql)
+- [123-more-model-categories.sql](conf/evolutions/123-more-model-categories.sql)

app/controllers/AiModelController.scala

@@ -57,6 +57,16 @@ object UpdateAiModelParameters {
   implicit val jsonFormat: OFormat[UpdateAiModelParameters] = Json.format[UpdateAiModelParameters]
 }
 
+case class RegisterAiModelParameters(id: ObjectId, // must be a valid MongoDB ObjectId
+                                     dataStoreName: String,
+                                     name: String,
+                                     comment: Option[String],
+                                     category: Option[AiModelCategory])
+
+object RegisterAiModelParameters {
+  implicit val jsonFormat: OFormat[RegisterAiModelParameters] = Json.format[RegisterAiModelParameters]
+}
+
 class AiModelController @Inject()(
     aiModelDAO: AiModelDAO,
     aiModelService: AiModelService,
@@ -209,6 +219,28 @@ class AiModelController @Inject()(
       } yield Ok(jsResult)
     }
 
+  def registerAiModel: Action[RegisterAiModelParameters] =
+    sil.SecuredAction.async(validateJson[RegisterAiModelParameters]) { implicit request =>
+      for {
+        _ <- userService.assertIsSuperUser(request.identity)
+        _ <- dataStoreDAO.findOneByName(request.body.dataStoreName) ?~> "dataStore.notFound"
+        _ <- aiModelDAO.findOne(request.body.id).reverse ?~> "aiModel.id.taken"
+        _ <- aiModelDAO.findOneByName(request.body.name).reverse ?~> "aiModel.name.taken"
+        _ <- aiModelDAO.insertOne(
+          AiModel(
+            request.body.id,
+            _organization = request.identity._organization,
+            request.body.dataStoreName,
+            request.identity._id,
+            None,
+            List.empty,
+            request.body.name,
+            request.body.comment,
+            request.body.category
+          ))
+      } yield Ok
+    }
+
   def deleteAiModel(aiModelId: String): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {

app/models/aimodels/AiModel.scala

@@ -144,4 +144,11 @@ class AiModelDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
         q"UPDATE webknossos.aiModels SET name = ${a.name}, comment = ${a.comment}, modified = ${a.modified} WHERE _id = ${a._id}".asUpdate)
     } yield ()
 
+  def findOneByName(name: String)(implicit ctx: DBAccessContext): Fox[AiModel] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(q"SELECT $columns FROM $existingCollectionName WHERE name = $name AND $accessQuery".as[AimodelsRow])
+      parsed <- parseFirst(r, name)
+    } yield parsed
+
 }

app/models/aimodels/AiModelCategory.scala

@@ -4,5 +4,5 @@ import com.scalableminds.util.enumeration.ExtendedEnumeration
 
 object AiModelCategory extends ExtendedEnumeration {
   type AiModelCategory = Value
-  val em_neurons, em_nuclei = Value
+  val em_neurons, em_nuclei, em_synapses, em_neuron_types, em_cell_organelles = Value
 }

app/utils/sql/SQLDAO.scala

@@ -47,7 +47,7 @@ abstract class SQLDAO[C, R, X <: AbstractTable[R]] @Inject()(sqlClient: SqlClien
       case Some(r) =>
         parse(r) ?~> ("sql: could not parse database row for object" + id)
       case _ =>
-        Fox.failure("sql: could not find object " + id)
+        Fox.empty
     }.flatten
 
   @nowarn // suppress warning about unused implicit ctx, as it is used in subclasses

conf/evolutions/123-more-model-categories.sql

@@ -0,0 +1,11 @@
+
+-- no transaction here, since ALTER TYPE ... ADD cannot run inside a transaction block
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 122, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_synapses';
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_neuron_types';
+ALTER TYPE webknossos.AI_MODEL_CATEGORY ADD VALUE 'em_cell_organelles';
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 123;
+

conf/evolutions/reversions/123-more-model-categories.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 123, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+-- removing enum values is not supported in postgres, see https://www.postgresql.org/docs/current/datatype-enum.html#DATATYPE-ENUM-IMPLEMENTATION-DETAILS
+
+UPDATE webknossos.aiModels SET isDeleted = TRUE WHERE category IN ('em_synapses', 'em_neuron_types', 'em_cell_organelles');
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 122;
+
+COMMIT TRANSACTION;

conf/webknossos.latest.routes

@@ -283,6 +283,7 @@ POST          /aiModels/inferences/runInference
 GET           /aiModels/inferences/:id                                                          controllers.AiModelController.readAiInferenceInfo(id: String)
 GET           /aiModels/inferences                                                              controllers.AiModelController.listAiInferences
 GET           /aiModels                                                                         controllers.AiModelController.listAiModels
+POST          /aiModels/register                                                                controllers.AiModelController.registerAiModel
 GET           /aiModels/:id                                                                     controllers.AiModelController.readAiModelInfo(id: String)
 PUT           /aiModels/:id                                                                     controllers.AiModelController.updateAiModelInfo(id: String)
 DELETE        /aiModels/:id                                                                     controllers.AiModelController.deleteAiModel(id: String)

tools/postgres/schema.sql

@@ -20,7 +20,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(122);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(123);
 COMMIT TRANSACTION;
 
 
@@ -546,7 +546,7 @@ CREATE TABLE webknossos.emailVerificationKeys(
   isUsed BOOLEAN NOT NULL DEFAULT false
 );
 
-CREATE TYPE webknossos.AI_MODEL_CATEGORY AS ENUM ('em_neurons', 'em_nuclei');
+CREATE TYPE webknossos.AI_MODEL_CATEGORY AS ENUM ('em_neurons', 'em_nuclei', 'em_synapses', 'em_neuron_types', 'em_cell_organelles');
 
 CREATE TABLE webknossos.aiModels(
   _id CHAR(24) PRIMARY KEY,