Use new mesh API for animation job (#7692)

* updated animation job to use proxy mesh infromation for new mesh api

* added ad-hoc meshes to animation job

* pass fallback layer and adhoc meshing options to worker

* upadted changelog

* updated wk docs

* apply PR feedback

* apply PR feedback 2

* reverted colorLayerName to layerName

* use tracing id for animations

* remove view mode parameter from animations

* also store mappingName for precomputed mesh info objects

* fix animation modal error message for DS without color layers

---------

Co-authored-by: Philipp Otto <[email protected]>

Author: hotzenklotz

CHANGELOG.unreleased.md

@@ -25,6 +25,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Changed some internal APIs to use spelling dataset instead of dataSet. This requires all connected datastores to be the latest version. [#7690](https://github.com/scalableminds/webknossos/pull/7690)
 - Toasts are shown until WEBKNOSSOS is running in the active browser tab again. Also, the content of most toasts that show errors or warnings is printed to the browser's console. [#7741](https://github.com/scalableminds/webknossos/pull/7741)
 - Improved UI speed when editing the description of an annotation. [#7769](https://github.com/scalableminds/webknossos/pull/7769)
+- Updated dataset animations to use the new meshing API. Animitation now support ad-hoc meshes and mappings. [#7692](https://github.com/scalableminds/webknossos/pull/7692) 
+
 
 ### Fixed
 - Fixed that the Command modifier on MacOS wasn't treated correctly for some shortcuts. Also, instead of the Alt key, the ⌥ key is shown as a hint in the status bar on MacOS. [#7659](https://github.com/scalableminds/webknossos/pull/7659)

app/controllers/JobsController.scala

@@ -34,9 +34,7 @@ case class AnimationJobOptions(
     layerName: String,
     boundingBox: BoundingBox,
     includeWatermark: Boolean,
-    segmentationLayerName: Option[String],
-    meshFileName: Option[String],
-    meshSegmentIds: Array[Int],
+    meshes: JsValue,
     movieResolution: MovieResolutionSetting.Value,
     cameraPosition: CameraPositionSetting.Value,
     intensityMin: Double,
@@ -406,19 +404,16 @@ class JobsController @Inject()(
           }
           layerName = animationJobOptions.layerName
           _ <- datasetService.assertValidLayerNameLax(layerName)
-          _ <- Fox.runOptional(animationJobOptions.segmentationLayerName)(datasetService.assertValidLayerNameLax)
           exportFileName = s"webknossos_animation_${formatDateForFilename(new Date())}__${datasetName}__$layerName.mp4"
           command = JobCommand.render_animation
           commandArgs = Json.obj(
             "organization_name" -> organizationName,
             "dataset_name" -> datasetName,
             "export_file_name" -> exportFileName,
             "layer_name" -> animationJobOptions.layerName,
-            "segmentation_layer_name" -> animationJobOptions.segmentationLayerName,
             "bounding_box" -> animationJobOptions.boundingBox.toLiteral,
             "include_watermark" -> animationJobOptions.includeWatermark,
-            "mesh_segment_ids" -> animationJobOptions.meshSegmentIds,
-            "meshfile_name" -> animationJobOptions.meshFileName,
+            "meshes" -> animationJobOptions.meshes,
             "movie_resolution" -> animationJobOptions.movieResolution,
             "camera_position" -> animationJobOptions.cameraPosition,
             "intensity_min" -> animationJobOptions.intensityMin,

docs/animations.md

@@ -9,7 +9,7 @@ A picture is worth a thousand words. In this spirit, you can use WEBKNOSSOS to c
 Creating an animation is easy:
 
 1. Open any dataset or annotation that you want to use for your animation. 
-2. Optionally, load any [pre-computed 3D meshes](./mesh_visualization.md#pre-computed-mesh-generation) for any segments that you wish to highlight.
+2. Optionally, load some [3D meshes](./mesh_visualization.md) for any segments that you wish to highlight.
 3. For larger datasets, use the bounding box tool to create a bounding box around your area of interest. Smaller datasets can be used in their entirety.
 4. From the `Menu` dropdown in navbar at the top of the screen, select "Create Animation".
 5. Configure the animation options as desired, i.e. camera movement or resolution.

frontend/javascripts/oxalis/model/actions/annotation_actions.ts

@@ -307,6 +307,7 @@ export const addPrecomputedMeshAction = (
   seedPosition: Vector3,
   seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null,
   meshFileName: string,
+  mappingName: string | null | undefined,
 ) =>
   ({
     type: "ADD_PRECOMPUTED_MESH",
@@ -315,6 +316,7 @@ export const addPrecomputedMeshAction = (
     seedPosition,
     seedAdditionalCoordinates,
     meshFileName,
+    mappingName,
   }) as const;
 
 export const setOthersMayEditForAnnotationAction = (othersMayEdit: boolean) =>

frontend/javascripts/oxalis/model/reducers/annotation_reducer.ts

@@ -321,8 +321,14 @@ function AnnotationReducer(state: OxalisState, action: Action): OxalisState {
     }
 
     case "ADD_PRECOMPUTED_MESH": {
-      const { layerName, segmentId, seedPosition, seedAdditionalCoordinates, meshFileName } =
-        action;
+      const {
+        layerName,
+        segmentId,
+        seedPosition,
+        seedAdditionalCoordinates,
+        meshFileName,
+        mappingName,
+      } = action;
       const meshInfo: MeshInformation = {
         segmentId: segmentId,
         seedPosition,
@@ -331,6 +337,7 @@ function AnnotationReducer(state: OxalisState, action: Action): OxalisState {
         isVisible: true,
         isPrecomputed: true,
         meshFileName,
+        mappingName,
       };
       const additionalCoordinates = state.flycam.additionalCoordinates;
       const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates);

frontend/javascripts/oxalis/model/sagas/mesh_saga.ts

@@ -746,8 +746,16 @@ function* loadPrecomputedMeshForSegmentId(
   segmentationLayer: APISegmentationLayer,
 ): Saga<void> {
   const layerName = segmentationLayer.name;
+  const mappingName = yield* call(getMappingName, segmentationLayer);
   yield* put(
-    addPrecomputedMeshAction(layerName, id, seedPosition, seedAdditionalCoordinates, meshFileName),
+    addPrecomputedMeshAction(
+      layerName,
+      id,
+      seedPosition,
+      seedAdditionalCoordinates,
+      meshFileName,
+      mappingName,
+    ),
   );
   yield* put(startedLoadingMeshAction(layerName, id));
   const dataset = yield* select((state) => state.dataset);
@@ -813,6 +821,18 @@ function* loadPrecomputedMeshForSegmentId(
   yield* put(finishedLoadingMeshAction(layerName, id));
 }
 
+function* getMappingName(segmentationLayer: APISegmentationLayer) {
+  const meshExtraInfo = yield* call(getMeshExtraInfo, segmentationLayer.name, null);
+  const editableMapping = yield* select((state) =>
+    getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId),
+  );
+
+  // meshExtraInfo.mappingName contains the currently active mapping
+  // (can be the id of an editable mapping). However, we always need to
+  // use the mapping name of the on-disk mapping.
+  return editableMapping != null ? editableMapping.baseMappingName : meshExtraInfo.mappingName;
+}
+
 function* _getChunkLoadingDescriptors(
   id: number,
   dataset: APIDataset,
@@ -826,19 +846,14 @@ function* _getChunkLoadingDescriptors(
   const { segmentMeshController } = getSceneController();
   const version = meshFile.formatVersion;
   const { meshFileName } = meshFile;
-  const meshExtraInfo = yield* call(getMeshExtraInfo, segmentationLayer.name, null);
 
   const editableMapping = yield* select((state) =>
     getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId),
   );
   const tracing = yield* select((state) =>
     getTracingForSegmentationLayer(state, segmentationLayer),
   );
-  const mappingName =
-    // meshExtraInfo.mappingName contains the currently active mapping
-    // (can be the id of an editable mapping). However, we always need to
-    // use the mapping name of the on-disk mapping.
-    editableMapping != null ? editableMapping.baseMappingName : meshExtraInfo.mappingName;
+  const mappingName = yield* call(getMappingName, segmentationLayer);
 
   if (version < 3) {
     console.warn("The active mesh file uses a version lower than 3, which is not supported");

frontend/javascripts/oxalis/store.ts

@@ -544,10 +544,10 @@ type BaseMeshInformation = {
   readonly seedAdditionalCoordinates?: AdditionalCoordinate[] | null;
   readonly isLoading: boolean;
   readonly isVisible: boolean;
+  readonly mappingName: string | null | undefined;
 };
 export type AdHocMeshInformation = BaseMeshInformation & {
   readonly isPrecomputed: false;
-  readonly mappingName: string | null | undefined;
   readonly mappingType: MappingType | null | undefined;
 };
 export type PrecomputedMeshInformation = BaseMeshInformation & {

frontend/javascripts/oxalis/view/action-bar/create_animation_modal.tsx

@@ -11,6 +11,7 @@ import {
   getColorLayers,
   getEffectiveIntensityRange,
   getLayerByName,
+  getResolutionInfo,
   is2dDataset,
 } from "oxalis/model/accessors/dataset_accessor";
 import {
@@ -24,6 +25,7 @@ import {
   MOVIE_RESOLUTIONS,
   APIDataLayer,
   APIJobType,
+  APISegmentationLayer,
 } from "types/api_flow_types";
 import { InfoCircleOutlined } from "@ant-design/icons";
 import { PricingEnforcedSpan } from "components/pricing_enforcers";
@@ -33,8 +35,8 @@ import {
 } from "admin/organization/pricing_plan_utils";
 import { BoundingBoxType, Vector3 } from "oxalis/constants";
 import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box";
-import { Model } from "oxalis/singletons";
 import { BoundingBoxSelection, LayerSelection } from "./starting_job_modals";
+import { getAdditionalCoordinatesAsString } from "oxalis/model/accessors/flycam_accessor";
 
 type Props = {
   isOpen: boolean;
@@ -75,12 +77,19 @@ function selectMagForTextureCreation(
   return [bestMag, bestDifference];
 }
 
-export function CreateAnimationModalWrapper(props: Props) {
+export default function CreateAnimationModalWrapper(props: Props) {
   const dataset = useSelector((state: OxalisState) => state.dataset);
 
   // early stop if no color layer exists
   const colorLayers = getColorLayers(dataset);
-  if (colorLayers.length === 0) return null;
+  if (colorLayers.length === 0) {
+    const { isOpen, onClose } = props;
+    return (
+      <Modal open={isOpen} onOk={onClose} onCancel={onClose} title="Create Animation">
+        WEBKNOSSOS cannot create animations for datasets without color layers.
+      </Modal>
+    );
+  }
 
   return <CreateAnimationModal {...props} />;
 }
@@ -128,7 +137,7 @@ function CreateAnimationModal(props: Props) {
   const validateAnimationOptions = (
     colorLayer: APIDataLayer,
     selectedBoundingBox: BoundingBoxType,
-    meshSegmentIds: number[],
+    meshes: Partial<MeshInformation>[],
   ) => {
     //  Validate the select parameters and dataset to make sure it actually works and does not overload the server
 
@@ -151,7 +160,7 @@ function CreateAnimationModal(props: Props) {
       !is2dDataset(state.dataset) && (colorLayer.additionalAxes?.length || 0) === 0;
     if (isDataset3D) errorMessages.push("Sorry, animations are only supported for 3D datasets.");
 
-    const isTooManyMeshes = meshSegmentIds.length > MAX_MESHES_PER_ANIMATION;
+    const isTooManyMeshes = meshes.length > MAX_MESHES_PER_ANIMATION;
     if (isTooManyMeshes)
       errorMessages.push(
         `You selected too many meshes for the animation. Please keep the number of meshes below ${MAX_MESHES_PER_ANIMATION} to create an animation.`,
@@ -171,32 +180,33 @@ function CreateAnimationModal(props: Props) {
       (bb) => bb.id === selectedBoundingBoxId,
     )!.boundingBox;
 
-    // Submit currently visible pre-computed meshes
-    let meshSegmentIds: number[] = [];
-    let meshFileName: string | undefined;
-    let segmentationLayerName: string | undefined;
-
-    const visibleSegmentationLayer = Model.getVisibleSegmentationLayer();
-
-    if (visibleSegmentationLayer) {
-      const availableMeshes = state.localSegmentationData[visibleSegmentationLayer.name].meshes;
-      if (availableMeshes == null) {
-        throw new Error("There is no mesh data in localSegmentationData.");
-      }
-      meshSegmentIds = Object.values(availableMeshes as Record<number, MeshInformation>)
-        .filter((mesh) => mesh.isVisible && mesh.isPrecomputed)
-        .map((mesh) => mesh.segmentId);
-
-      const currentMeshFile =
-        state.localSegmentationData[visibleSegmentationLayer.name].currentMeshFile;
-      meshFileName = currentMeshFile?.meshFileName;
-
-      if (visibleSegmentationLayer.fallbackLayerInfo) {
-        segmentationLayerName = visibleSegmentationLayer.fallbackLayerInfo.name;
-      } else {
-        segmentationLayerName = visibleSegmentationLayer.name;
-      }
-    }
+    // Submit currently visible pre-computed & ad-hoc meshes
+    const axis = getAdditionalCoordinatesAsString([]);
+    const layerNames = Object.keys(state.localSegmentationData);
+    const { preferredQualityForMeshAdHocComputation } = state.temporaryConfiguration;
+
+    const meshes: RenderAnimationOptions["meshes"] = layerNames.flatMap((layerName) => {
+      const meshInfos = state.localSegmentationData[layerName]?.meshes?.[axis] || {};
+
+      const layer = getLayerByName(state.dataset, layerName) as APISegmentationLayer;
+      const fullLayerName = layer.fallbackLayerInfo?.name || layerName;
+
+      const adhocMagIndex = getResolutionInfo(layer.resolutions).getClosestExistingIndex(
+        preferredQualityForMeshAdHocComputation,
+      );
+      const adhocMag = layer.resolutions[adhocMagIndex];
+
+      return Object.values(meshInfos)
+        .filter((meshInfo: MeshInformation) => meshInfo.isVisible)
+        .flatMap((meshInfo: MeshInformation) => {
+          return {
+            layerName: fullLayerName,
+            tracingId: layer.tracingId || null,
+            adhocMag,
+            ...meshInfo,
+          };
+        });
+    });
 
     // Submit the configured min/max intensity info to support float datasets
     const [intensityMin, intensityMax] = getEffectiveIntensityRange(
@@ -209,9 +219,7 @@ function CreateAnimationModal(props: Props) {
 
     const animationOptions: RenderAnimationOptions = {
       layerName: selectedColorLayerName,
-      segmentationLayerName,
-      meshFileName,
-      meshSegmentIds,
+      meshes,
       intensityMin,
       intensityMax,
       magForTextures,
@@ -221,7 +229,7 @@ function CreateAnimationModal(props: Props) {
       cameraPosition: selectedCameraPosition,
     };
 
-    if (!validateAnimationOptions(colorLayer, boundingBox, meshSegmentIds)) return;
+    if (!validateAnimationOptions(colorLayer, boundingBox, meshes)) return;
 
     startRenderAnimationJob(state.dataset.owningOrganization, state.dataset.name, animationOptions);
 
@@ -338,7 +346,7 @@ function CreateAnimationModal(props: Props) {
               >
                 Include the currently selected 3D meshes
                 <Tooltip
-                  title="When enabled, all (pre-computed) meshes currently visible in WEBKNOSSOS will be included in the animation."
+                  title="When enabled, all meshes currently visible in WEBKNOSSOS will be included in the animation."
                   placement="right"
                 >
                   <InfoCircleOutlined style={{ marginLeft: 10 }} />
@@ -401,5 +409,3 @@ function CreateAnimationModal(props: Props) {
     </Modal>
   );
 }
-
-export default CreateAnimationModal;

frontend/javascripts/oxalis/view/action-bar/tracing_actions_view.tsx

@@ -74,7 +74,7 @@ import UrlManager from "oxalis/controller/url_manager";
 import { withAuthentication } from "admin/auth/authentication_modal";
 import { PrivateLinksModal } from "./private_links_view";
 import { ItemType, SubMenuType } from "antd/lib/menu/hooks/useItems";
-import { CreateAnimationModalWrapper as CreateAnimationModal } from "./create_animation_modal";
+import CreateAnimationModal from "./create_animation_modal";
 
 const AsyncButtonWithAuthentication = withAuthentication<AsyncButtonProps, typeof AsyncButton>(
   AsyncButton,

frontend/javascripts/oxalis/view/action-bar/view_dataset_actions_view.tsx

@@ -19,7 +19,7 @@ import {
 import Store, { OxalisState } from "oxalis/store";
 import { MenuItemType, SubMenuType } from "antd/lib/menu/hooks/useItems";
 import DownloadModalView from "./download_modal_view";
-import { CreateAnimationModalWrapper as CreateAnimationModal } from "./create_animation_modal";
+import CreateAnimationModal from "./create_animation_modal";
 
 type Props = {
   layoutMenu: SubMenuType;