Zarr private links

CHANGELOG.unreleased.md

@@ -22,6 +22,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added a "duplicate" button for annotations. [#6386](https://github.com/scalableminds/webknossos/pull/6386)
 - Added new filter options for the dataset list api at `api/datasets`: `organizationName: String`, `onlyMyOrganization: Boolean`, `uploaderId: String` [#6377](https://github.com/scalableminds/webknossos/pull/6377)
 - The Add Dataset view now hosts an Add Zarr Dataset tab to explore, configure and import remote Zarr datasets. [#6335](https://github.com/scalableminds/webknossos/pull/6335)
+- Added a UI to manage Zarr links for an annotation so that the annotation can be streamed to 3rd party tools. This change also includes new backend API routes for using the (private) zarr links to annotations as well as creating them. [#6367](https://github.com/scalableminds/webknossos/pull/6367)
 
 ### Changed
 - webKnossos uses WebGL 2 instead of WebGL 1 now. In case your browser/hardware does not support this, webKnossos will alert you and you need to upgrade your system. [#6350](https://github.com/scalableminds/webknossos/pull/6350)

MIGRATIONS.unreleased.md

@@ -15,3 +15,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 - [084-annotation-contributors.sql](conf/evolutions/084-annotation-contributors.sql)
 - [085-add-annotations-publicationforeignkey](conf/evolutions/085-add-annotations-publicationforeignkey.sql)
 - [086-drop-foreign-datastores.sql](conf/evolutions/086-drop-foreign-datastores.sql.sql)
+- [087-zarr-private-links](conf/evolutions/087-zarr-private-links.sql)

app/controllers/AnnotationController.scala

@@ -5,9 +5,14 @@ import com.mohiva.play.silhouette.api.Silhouette
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.geometry.BoundingBox
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions
 import com.scalableminds.webknossos.tracingstore.tracings.{TracingIds, TracingType}
 import io.swagger.annotations._
+
+import javax.inject.Inject
+import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent}
+import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType
 import models.annotation.AnnotationState.Cancelled
 import models.annotation._
 import models.binary.{DataSetDAO, DataSetService}
@@ -18,13 +23,12 @@ import models.user.time._
 import models.user.{User, UserDAO, UserService}
 import oxalis.security.{URLSharing, WkEnv}
 import play.api.i18n.{Messages, MessagesProvider}
-import play.api.libs.json.{JsArray, _}
+import play.api.libs.json._
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import utils.{ObjectId, WkConf}
 
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent}
-import models.annotation.AnnotationLayerType.AnnotationLayerType
 import models.organization.OrganizationDAO
 import oxalis.mail.{MailchimpClient, MailchimpTag}
 
@@ -53,6 +57,7 @@ class AnnotationController @Inject()(
     teamService: TeamService,
     projectDAO: ProjectDAO,
     teamDAO: TeamDAO,
+    annotationPrivateLinkDAO: AnnotationPrivateLinkDAO,
     timeSpanService: TimeSpanService,
     annotationMerger: AnnotationMerger,
     tracingStoreService: TracingStoreService,

app/controllers/AnnotationIOController.scala

@@ -2,7 +2,6 @@ package controllers
 
 import java.io.{BufferedOutputStream, File, FileOutputStream}
 import java.util.zip.Deflater
-
 import akka.actor.ActorSystem
 import akka.stream.Materializer
 import com.mohiva.play.silhouette.api.Silhouette
@@ -12,6 +11,11 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits, TextUtils}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings}
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
+import com.scalableminds.webknossos.datastore.models.annotation.{
+  AnnotationLayer,
+  AnnotationLayerType,
+  FetchedAnnotationLayer
+}
 import com.scalableminds.webknossos.datastore.models.datasource.{
   AbstractSegmentationLayer,
   DataLayerLike,
@@ -22,6 +26,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.TracingType
 import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingDefaults
 import com.typesafe.scalalogging.LazyLogging
 import io.swagger.annotations._
+
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, DownloadAnnotationEvent, UploadAnnotationEvent}
 import models.annotation.AnnotationState._

app/controllers/AnnotationPrivateLinkController.scala

@@ -0,0 +1,174 @@
+package controllers
+
+import com.mohiva.play.silhouette.api.Silhouette
+import com.scalableminds.util.accesscontext.GlobalAccessContext
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.tools.FoxImplicits
+import io.swagger.annotations._
+import play.api.libs.json._
+
+import javax.inject.Inject
+import models.annotation._
+import oxalis.security.WkEnv
+import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
+import utils.ObjectId
+
+import scala.concurrent.ExecutionContext
+@Api
+class AnnotationPrivateLinkController @Inject()(
+    annotationDAO: AnnotationDAO,
+    annotationService: AnnotationService,
+    annotationPrivateLinkDAO: AnnotationPrivateLinkDAO,
+    annotationPrivateLinkService: AnnotationPrivateLinkService,
+    sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, val bodyParsers: PlayBodyParsers)
+    extends Controller
+    with FoxImplicits {
+
+  @ApiOperation(hidden = true, value = "")
+  def lookupPrivateLink(accessToken: String): Action[AnyContent] = Action.async { implicit request =>
+    for {
+      annotationPrivateLink <- annotationPrivateLinkDAO.findOneByAccessToken(accessToken)
+      _ <- bool2Fox(annotationPrivateLink.expirationDateTime.forall(_ > System.currentTimeMillis())) ?~> "Token expired" ~> 404
+      annotation: Annotation <- annotationDAO.findOne(annotationPrivateLink._annotation)(GlobalAccessContext)
+      writtenAnnotation <- annotationService.writesAsAnnotationSource(annotation)
+    } yield Ok(writtenAnnotation)
+  }
+
+  @ApiOperation(value = "List all existing private zarr links for a user", nickname = "listPrivateLinks")
+  @ApiResponses(
+    Array(new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+          new ApiResponse(code = 400, message = badRequestLabel)))
+  def list: Action[AnyContent] = sil.SecuredAction.async { implicit request =>
+    for {
+      links <- annotationPrivateLinkDAO.findAll
+      linksJsonList <- Fox.serialCombined(links)(annotationPrivateLinkService.publicWrites)
+    } yield Ok(Json.toJson(linksJsonList))
+  }
+
+  @ApiOperation(value = "List all existing private zarr links for a user for a given annotation",
+                nickname = "listPrivateLinksByAnnotation")
+  @ApiResponses(
+    Array(new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+          new ApiResponse(code = 400, message = badRequestLabel)))
+  def listByAnnotation(@ApiParam(value = "The id of the annotation") annotationId: String): Action[AnyContent] =
+    sil.SecuredAction.async { implicit request =>
+      for {
+        annotationIdValidated <- ObjectId.fromString(annotationId)
+        links <- annotationPrivateLinkDAO.findAllByAnnotation(annotationIdValidated)
+        linksJsonList <- Fox.serialCombined(links)(annotationPrivateLinkService.publicWrites)
+      } yield Ok(Json.toJson(linksJsonList))
+    }
+
+  @ApiOperation(value = "Get the private zarr link for a user for a given id if it exists", nickname = "getPrivateLink")
+  @ApiResponses(
+    Array(
+      new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+      new ApiResponse(code = 400, message = badRequestLabel),
+      new ApiResponse(code = 404, message = badRequestLabel)
+    ))
+  def get(@ApiParam(value = "The id of the private link") id: String): Action[AnyContent] = sil.SecuredAction.async {
+    implicit request =>
+      for {
+        idValidated <- ObjectId.fromString(id)
+
+        annotationPrivateLink <- annotationPrivateLinkDAO.findOne(idValidated)
+        _ <- bool2Fox(annotationPrivateLink.expirationDateTime.forall(_ > System.currentTimeMillis())) ?~> "Token expired" ~> NOT_FOUND
+        _ <- annotationDAO.findOne(annotationPrivateLink._annotation) ?~> "annotation.notFound" ~> NOT_FOUND
+
+        annotationPrivateLinkJs <- annotationPrivateLinkService.publicWrites(annotationPrivateLink)
+      } yield Ok(annotationPrivateLinkJs)
+  }
+
+  @ApiOperation(
+    value = """Creates a given private link for an annotation for zarr streaming
+Expects:
+ - As JSON object body with keys:
+  - annotation (string): annotation id to create private link for
+  - expirationDateTime (Optional[bool]): optional UNIX timestamp, expiration date and time for the link
+""",
+    nickname = "createPrivateLink"
+  )
+  @ApiImplicitParams(
+    Array(
+      new ApiImplicitParam(name = "annotationPrivateLinkParams",
+                           required = true,
+                           dataTypeClass = classOf[AnnotationPrivateLinkParams],
+                           paramType = "body")))
+  @ApiResponses(
+    Array(
+      new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+      new ApiResponse(code = 400, message = badRequestLabel),
+      new ApiResponse(code = 404, message = badRequestLabel),
+      new ApiResponse(code = 403, message = badRequestLabel)
+    ))
+  def create: Action[AnnotationPrivateLinkParams] = sil.SecuredAction.async(validateJson[AnnotationPrivateLinkParams]) {
+    implicit request =>
+      val params = request.body
+      val _id = ObjectId.generate
+      val accessToken = annotationPrivateLinkService.generateToken
+      for {
+        annotationId <- ObjectId.fromString(params.annotation)
+        _ <- annotationDAO.assertUpdateAccess(annotationId) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- annotationPrivateLinkDAO.insertOne(
+          AnnotationPrivateLink(_id, annotationId, accessToken, params.expirationDateTime)) ?~> "create.failed"
+        inserted <- annotationPrivateLinkDAO.findOne(_id)
+        js <- annotationPrivateLinkService.publicWrites(inserted)
+      } yield Ok(js)
+  }
+
+  @ApiOperation(
+    value = """Updates a given private link for an annotation for zarr streaming
+Expects:
+ - As JSON object body with keys:
+  - annotation (string): annotation id to create private link for
+  - expirationDateTime (Optional[bool]): optional UNIX timestamp, expiration date and time for the link
+""",
+    nickname = "updatePrivateLink"
+  )
+  @ApiImplicitParams(
+    Array(
+      new ApiImplicitParam(name = "annotationPrivateLinkParams",
+                           required = true,
+                           dataTypeClass = classOf[AnnotationPrivateLinkParams],
+                           paramType = "body")))
+  @ApiResponses(
+    Array(
+      new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+      new ApiResponse(code = 400, message = badRequestLabel),
+      new ApiResponse(code = 404, message = badRequestLabel),
+      new ApiResponse(code = 403, message = badRequestLabel)
+    ))
+  def update(@ApiParam(value = "The id of the private link") id: String): Action[AnnotationPrivateLinkParams] =
+    sil.SecuredAction.async(validateJson[AnnotationPrivateLinkParams]) { implicit request =>
+      val params = request.body
+      for {
+        annotationId <- ObjectId.fromString(params.annotation)
+        idValidated <- ObjectId.fromString(id)
+        aPLInfo <- annotationPrivateLinkDAO.findOne(idValidated) ?~> "annotation private link not found" ~> NOT_FOUND
+        _ <- annotationDAO.assertUpdateAccess(aPLInfo._annotation) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- annotationDAO.assertUpdateAccess(annotationId) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- annotationPrivateLinkDAO.updateOne(idValidated, annotationId, params.expirationDateTime) ?~> "update.failed"
+        updated <- annotationPrivateLinkDAO.findOne(idValidated) ?~> "not Found"
+        js <- annotationPrivateLinkService.publicWrites(updated) ?~> "write failed"
+      } yield Ok(js)
+    }
+
+  @ApiOperation(value = "Deletes a given private link for an annotation for zarr streaming",
+                nickname = "deletePrivateLink")
+  @ApiResponses(
+    Array(
+      new ApiResponse(code = 200, message = "JSON object containing string that private link was deleted."),
+      new ApiResponse(code = 400, message = badRequestLabel),
+      new ApiResponse(code = 404, message = badRequestLabel),
+      new ApiResponse(code = 403, message = badRequestLabel)
+    ))
+  def delete(@ApiParam(value = "The id of the private link") id: String): Action[AnyContent] = sil.SecuredAction.async {
+    implicit request =>
+      for {
+        idValidated <- ObjectId.fromString(id)
+        aPLInfo <- annotationPrivateLinkDAO.findOne(idValidated) ?~> "notFound" ~> NOT_FOUND
+        _ <- annotationDAO.assertUpdateAccess(aPLInfo._annotation) ?~> "notAllowed" ~> FORBIDDEN
+        _ <- annotationPrivateLinkDAO.deleteOne(idValidated) ?~> "delete failed"
+      } yield JsonOk("privateLink deleted")
+  }
+}

app/controllers/LegacyApiController.scala

@@ -3,9 +3,9 @@ package controllers
 import com.mohiva.play.silhouette.api.Silhouette
 import com.mohiva.play.silhouette.api.actions.{SecuredRequest, UserAwareRequest}
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions
 import javax.inject.Inject
-import models.annotation.{AnnotationLayer, AnnotationLayerType}
 import models.project.ProjectDAO
 import models.task.{TaskDAO, TaskService}
 import models.user.User

app/controllers/UserTokenController.scala

@@ -34,13 +34,14 @@ object RpcTokenHolder {
    * The token is refreshed on every wK restart.
    * Keep it secret!
    */
-  lazy val webKnossosToken: String = CompactRandomIDGenerator.generateBlocking()
+  lazy val webKnossosToken: String = RandomIDGenerator.generateBlocking()
 }
 
 @Api
 class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
                                     dataSetService: DataSetService,
                                     annotationDAO: AnnotationDAO,
+                                    annotationPrivateLinkDAO: AnnotationPrivateLinkDAO,
                                     userService: UserService,
                                     organizationDAO: OrganizationDAO,
                                     annotationStore: AnnotationStore,
@@ -101,7 +102,7 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
           case AccessResourceType.datasource =>
             handleDataSourceAccess(accessRequest.resourceId, accessRequest.mode, userBox)(sharingTokenAccessCtx)
           case AccessResourceType.tracing =>
-            handleTracingAccess(accessRequest.resourceId.name, accessRequest.mode, userBox)
+            handleTracingAccess(accessRequest.resourceId.name, accessRequest.mode, userBox, token)
           case AccessResourceType.jobExport =>
             handleJobExportAccess(accessRequest.resourceId.name, accessRequest.mode, userBox)
           case _ =>
@@ -163,8 +164,12 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
     }
   }
 
-  private def handleTracingAccess(tracingId: String, mode: AccessMode, userBox: Box[User]): Fox[UserAccessAnswer] = {
+  private def handleTracingAccess(tracingId: String,
+                                  mode: AccessMode,
+                                  userBox: Box[User],
+                                  token: Option[String]): Fox[UserAccessAnswer] = {
     // Access is explicitly checked by userBox, not by DBAccessContext, as there is no token sharing for annotations
+    // Optionally, a accessToken can be provided which explicitly looks up the read right the private link table
 
     def findAnnotationForTracing(tracingId: String)(implicit ctx: DBAccessContext): Fox[Annotation] = {
       val annotationFox = annotationDAO.findOneByTracingId(tracingId)
@@ -190,9 +195,16 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
     else {
       for {
         annotation <- findAnnotationForTracing(tracingId)(GlobalAccessContext) ?~> "annotation.notFound"
+        annotationAccessByToken <- token
+          .map(annotationPrivateLinkDAO.findOneByAccessToken)
+          .getOrElse(Fox.empty)
+          .futureBox
+
+        allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation)
         restrictions <- annotationInformationProvider.restrictionsFor(
           AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound"
-        allowed <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck"
+        allowedByUser <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck"
+        allowed = allowedByToken || allowedByUser
       } yield {
         if (allowed) UserAccessAnswer(granted = true)
         else UserAccessAnswer(granted = false, Some(s"No ${mode.toString} access to tracing"))

app/models/annotation/Annotation.scala

@@ -2,8 +2,10 @@ package models.annotation
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
+import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType}
 import com.scalableminds.webknossos.schema.Tables._
 import com.scalableminds.webknossos.tracingstore.tracings.TracingType
+
 import javax.inject.Inject
 import models.annotation.AnnotationState._
 import models.annotation.AnnotationType.AnnotationType

app/models/annotation/AnnotationMerger.scala

@@ -2,7 +2,9 @@ package models.annotation
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType}
 import com.typesafe.scalalogging.LazyLogging
+
 import javax.inject.Inject
 import models.annotation.AnnotationType.AnnotationType
 import models.binary.DataSetDAO