Merge branch 'master' into objectid-tracingstore

Author: fm3

conf/messages

@@ -149,6 +149,7 @@ zarr.invalidFirstChunkCoord=First Channel must be 0
 zarr.chunkNotFound=Could not find the requested chunk
 zarr.notEnoughCoordinates=Invalid number of chunk coordinates. Expected to get at least 3 dimensions and channel 0.
 zarr.invalidAdditionalCoordinates=Invalid additional coordinates for this data layer.
+zarr.readShardIndex.failed=Failed to read shard information for zarr data. This may indicate missing data.
 
 nml.file.uploadSuccess=Successfully uploaded file
 nml.file.notFound=Could not extract NML file

frontend/javascripts/admin/datastore_health_check.ts

@@ -1,22 +1,19 @@
 import * as RestAPI from "admin/rest_api";
-import Request from "libs/request";
 import Toast from "libs/toast";
 import _ from "lodash";
 import messages from "messages";
+import type { APIBuildInfoDatastore, APIBuildInfoWk } from "types/api_types";
 
 // Create a throttled function which depends on its arguments.
 // That way, each datastore is checked for health in a throttled and isolated manner
-// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'func' implicitly has an 'any' type.
-const memoizedThrottle = (func, wait = 0, options = {}): ((...args: Array<any>) => any) => {
+const memoizedThrottle = <F extends (...args: Array<any>) => any>(func: F, wait = 0): F => {
   // Memoize the creation of a throttling function
-  // @ts-expect-error ts-migrate(2339) FIXME: Property 'resolver' does not exist on type '{}'.
-  const mem = _.memoize(() => _.throttle(func, wait, options), options.resolver);
+  const mem = _.memoize((..._args: any[]) => _.throttle(func, wait));
 
-  return (...args: Array<any>) => {
+  return ((...args: Parameters<F>) => {
     // look up (or create) the throttling function and invoke it
-    // @ts-expect-error ts-migrate(2556) FIXME: Expected 0 arguments, but got 1 or more.
-    mem(...args)(...args);
-  };
+    return mem(...args)(...args);
+  }) as F;
 };
 
 // Do not call this function directly, but call pingMentionedDataStores instead
@@ -34,10 +31,10 @@ const pingDataStoreIfAppropriate = memoizedThrottle(async (requestedUrl: string)
     return;
   }
 
-  const stores = [
+  const stores: Array<{ url: string; path: "tracings" | "data" }> = [
     { ...tracingstore, path: "tracings" },
-    ...datastores.map((datastore) => ({ ...datastore, path: "data" })),
-  ];
+    ...datastores.map((datastore) => ({ ...datastore, path: "data" as const })),
+  ] as const;
 
   if (isInMaintenance) {
     Toast.warning(messages.planned_maintenance);
@@ -46,13 +43,14 @@ const pingDataStoreIfAppropriate = memoizedThrottle(async (requestedUrl: string)
 
     if (usedStore != null) {
       const { url, path } = usedStore;
-      const healthEndpoint = `${url}/${path}/health`;
-      Request.triggerRequest(healthEndpoint, {
-        doNotInvestigate: true,
-        mode: "cors",
-        timeout: 5000,
-      }).then(
-        () => checkVersionMismatch(url),
+      RestAPI.pingHealthEndpoint(url, path).then(
+        () => {
+          if (usedStore.path === "data") {
+            // Only check a version mismatch for the data store, because
+            // the tracingstore doesn't serve a tracingstoreApiVersion field.
+            checkVersionMismatchInDataStore(url);
+          }
+        },
         () =>
           Toast.warning(
             messages["datastore.health"]({
@@ -64,15 +62,13 @@ const pingDataStoreIfAppropriate = memoizedThrottle(async (requestedUrl: string)
   }
 }, 5000);
 
-async function checkVersionMismatch(url: string) {
-  const [buildinfoWebknossos, buildinfoDatastore] = await Promise.all([
+async function checkVersionMismatchInDataStore(datastoreUrl: string) {
+  const [buildinfoWebknossos, buildinfoDatastore] = (await Promise.all([
     RestAPI.getBuildInfo(),
-    RestAPI.getDataStoreBuildInfo(url),
-  ]);
+    RestAPI.getDataOrTracingStoreBuildInfo(datastoreUrl),
+  ])) as [APIBuildInfoWk, APIBuildInfoDatastore];
   const expectedDatastoreApiVersion = buildinfoWebknossos.webknossos.datastoreApiVersion;
-  const buildInfoWebknossosDatastore = buildinfoDatastore.webknossosDatastore
-    ? buildinfoDatastore.webknossosDatastore
-    : buildinfoDatastore.webknossos;
+  const buildInfoWebknossosDatastore = buildinfoDatastore.webknossosDatastore;
   const suppliedDatastoreApiVersion = buildInfoWebknossosDatastore.datastoreApiVersion;
 
   if (
@@ -83,7 +79,7 @@ async function checkVersionMismatch(url: string) {
       messages["datastore.version.too_new"]({
         expectedDatastoreApiVersion,
         suppliedDatastoreApiVersion,
-        url,
+        datastoreUrl,
       }),
     );
   } else if (
@@ -94,7 +90,7 @@ async function checkVersionMismatch(url: string) {
       messages["datastore.version.too_old"]({
         expectedDatastoreApiVersion,
         suppliedDatastoreApiVersion,
-        url,
+        datastoreUrl,
       }),
     );
   }

frontend/javascripts/admin/rest_api.ts

@@ -15,7 +15,9 @@ import {
   type APIAnnotationType,
   type APIAnnotationVisibility,
   type APIAvailableTasksReport,
-  type APIBuildInfo,
+  type APIBuildInfoDatastore,
+  type APIBuildInfoTracingstore,
+  type APIBuildInfoWk,
   type APICompoundType,
   type APIConnectomeFile,
   type APIDataSource,
@@ -1789,17 +1791,31 @@ export async function getPricingPlanStatus(): Promise<APIPricingPlanStatus> {
 
 export const cachedGetPricingPlanStatus = _.memoize(getPricingPlanStatus);
 
+// ### Health
+export function pingHealthEndpoint(url: string, path: "tracings" | "data"): Promise<void> {
+  const healthEndpoint = `${url}/${path}/health`;
+  return Request.triggerRequest(healthEndpoint, {
+    doNotInvestigate: true,
+    mode: "cors",
+    timeout: 5000,
+  });
+}
+
 // ### BuildInfo webknossos
-export function getBuildInfo(): Promise<APIBuildInfo> {
+export function getBuildInfo(): Promise<APIBuildInfoWk> {
   return Request.receiveJSON("/api/buildinfo", {
     doNotInvestigate: true,
+    mode: "cors",
   });
 }
 
-// ### BuildInfo datastore
-export function getDataStoreBuildInfo(dataStoreUrl: string): Promise<APIBuildInfo> {
-  return Request.receiveJSON(`${dataStoreUrl}/api/buildinfo`, {
+// ### BuildInfo datastore/tracingstore
+export function getDataOrTracingStoreBuildInfo(
+  dataOrTracingStoreUrl: string,
+): Promise<APIBuildInfoDatastore | APIBuildInfoTracingstore> {
+  return Request.receiveJSON(`${dataOrTracingStoreUrl}/api/buildinfo`, {
     doNotInvestigate: true,
+    mode: "cors",
   });
 }
 

frontend/javascripts/libs/request.ts

@@ -1,4 +1,3 @@
-import { pingMentionedDataStores } from "admin/datastore_health_check";
 import handleStatus from "libs/handle_http_status";
 import Toast from "libs/toast";
 import _ from "lodash";
@@ -286,16 +285,17 @@ class Request {
       setTimeout(() => resolve("timeout"), timeout);
     });
 
-  handleError = (
+  handleError = async (
     requestedUrl: string,
     showErrorToast: boolean,
     doInvestigate: boolean,
     error: Response | Error,
   ): Promise<void> => {
     if (doInvestigate) {
+      // Avoid circular imports via dynamic import
+      const { pingMentionedDataStores } = await import("admin/datastore_health_check");
       // Check whether this request failed due to a problematic datastore
       pingMentionedDataStores(requestedUrl);
-
       if (error instanceof Response) {
         return error.text().then(
           (text) => {

frontend/javascripts/test/fixtures/build_info.ts

@@ -1,6 +1,6 @@
-import type { APIBuildInfo } from "types/api_types";
+import type { APIBuildInfoWk } from "types/api_types";
 
-export const buildInfo: APIBuildInfo = {
+export const buildInfo: APIBuildInfoWk = {
   webknossos: {
     name: "webknossos",
     ciTag: "",
@@ -21,6 +21,7 @@ export const buildInfo: APIBuildInfo = {
     sbtVersion: "1.4.1",
     builtAtString: "2021-12-20 21:30:36.569",
   },
+  httpApiVersioning: { currentApiVersion: 9, oldestSupportedApiVersion: 5 },
   schemaVersion: 101,
   localDataStoreEnabled: false,
   localTracingStoreEnabled: true,

frontend/javascripts/types/api_types.ts

@@ -680,7 +680,7 @@ export type APIPricingPlanStatus = {
   readonly isAlmostExceeded: boolean; // stays true when isExceeded is true)
 };
 
-export type APIBuildInfo = {
+export type APIBuildInfoWk = {
   webknossos: {
     name: string;
     commitHash: string;
@@ -702,7 +702,14 @@ export type APIBuildInfo = {
     sbtVersion: string;
     builtAtString: string;
   };
-  webknossosDatastore?: {
+  schemaVersion: number;
+  httpApiVersioning: { currentApiVersion: number; oldestSupportedApiVersion: number };
+  localDataStoreEnabled: boolean;
+  localTracingStoreEnabled: boolean;
+};
+
+export type APIBuildInfoDatastore = {
+  webknossosDatastore: {
     name: string;
     commitHash: string;
     scalaVersion: string;
@@ -711,13 +718,25 @@ export type APIBuildInfo = {
     commitDate: string;
     ciTag: string;
     ciBuild: string;
-    gitTag: string;
     datastoreApiVersion: string;
   };
-  schemaVersion: number;
-  localDataStoreEnabled: boolean;
-  localTracingStoreEnabled: boolean;
 };
+
+export type APIBuildInfoTracingstore = {
+  webknossosTracingstore: {
+    name: string;
+    commitHash: string;
+    scalaVersion: string;
+    version: string;
+    sbtVersion: string;
+    commitDate: string;
+    ciTag: string;
+    ciBuild: string;
+  };
+};
+
+export type APIBuildInfo = APIBuildInfoWk | APIBuildInfoDatastore | APIBuildInfoTracingstore;
+
 export type APIFeatureToggles = {
   readonly discussionBoard: string | false;
   readonly discussionBoardRequiresAdmin: boolean;

frontend/javascripts/viewer/model/helpers/nml_helpers.ts

@@ -6,7 +6,7 @@ import { location } from "libs/window";
 import _ from "lodash";
 import messages from "messages";
 import Saxophone from "saxophone";
-import type { APIBuildInfo, MetadataEntryProto } from "types/api_types";
+import type { APIBuildInfoWk, MetadataEntryProto } from "types/api_types";
 import type { AdditionalCoordinate } from "types/api_types";
 import {
   type BoundingBoxType,
@@ -136,7 +136,7 @@ export function serializeToNml(
   state: WebknossosState,
   annotation: StoreAnnotation,
   tracing: SkeletonTracing,
-  buildInfo: APIBuildInfo,
+  buildInfo: APIBuildInfoWk,
   applyTransform: boolean,
 ): string {
   // Only visible trees will be serialized!
@@ -166,7 +166,7 @@ export function serializeToNml(
 function serializeMetaInformation(
   state: WebknossosState,
   annotation: StoreAnnotation,
-  buildInfo: APIBuildInfo,
+  buildInfo: APIBuildInfoWk,
 ): string[] {
   return _.compact([
     serializeTag("meta", {

tools/check-cyclic-dependencies.js

@@ -3,23 +3,17 @@ const { parseDependencyTree, parseCircular } = dpdm;
 
 const KNOWN_CYCLES = [
   [
-    "frontend/javascripts/viewer/model/accessors/view_mode_accessor.ts",
     "frontend/javascripts/viewer/model/accessors/flycam_accessor.ts",
+    "frontend/javascripts/viewer/model/accessors/view_mode_accessor.ts",
   ],
   [
     "frontend/javascripts/viewer/model/accessors/flycam_accessor.ts",
     "frontend/javascripts/viewer/model/reducers/flycam_reducer.ts",
   ],
   [
-    "frontend/javascripts/viewer/view/right-border-tabs/tree_hierarchy_view_helpers.ts",
+    "frontend/javascripts/viewer/view/right-border-tabs/trees_tab/tree_hierarchy_view_helpers.ts",
     "frontend/javascripts/viewer/model/accessors/skeletontracing_accessor.ts",
   ],
-  ["frontend/javascripts/libs/request.ts", "frontend/javascripts/admin/datastore_health_check.ts"],
-  [
-    "frontend/javascripts/admin/rest_api.ts",
-    "frontend/javascripts/libs/request.ts",
-    "frontend/javascripts/admin/datastore_health_check.ts",
-  ],
   [
     "frontend/javascripts/viewer/view/action-bar/download_modal_view.tsx",
     "frontend/javascripts/viewer/view/action-bar/starting_job_modals.tsx",
@@ -41,15 +35,16 @@ const KNOWN_CYCLES = [
     "frontend/javascripts/dashboard/folders/folder_tree.tsx",
   ],
   [
-    "frontend/javascripts/viewer/model_initialization.ts",
     "frontend/javascripts/viewer/controller/url_manager.ts",
+    "frontend/javascripts/viewer/model_initialization.ts",
   ],
   [
     "frontend/javascripts/viewer/geometries/plane.ts",
     "frontend/javascripts/viewer/geometries/materials/plane_material_factory.ts",
     "frontend/javascripts/viewer/shaders/main_data_shaders.glsl.ts",
   ],
 ];
+
 parseDependencyTree("frontend/javascripts/main.tsx", {
   /* options, see below */
   extensions: [".ts", ".tsx"],

unreleased_changes/8633.md

@@ -0,0 +1,2 @@
+### Added
+- Agglomerate Mappings can now also be read from the new zarr3-based format, and from remote object storage.

unreleased_changes/8683.md

@@ -0,0 +1,2 @@
+### Fixed
+- Fixed that the healthcheck for a dedicated tracing store would fail sometimes.