Client-side stereo detection transfer via ONNX (first pass)

client/dive-common/apispec.ts

@@ -266,6 +266,95 @@ function useApi() {
   return use<Readonly<Api>>(ApiSymbol);
 }
 
+/**
+ * Interactive Segmentation Types
+ */
+export interface SegmentationPredictRequest {
+  /** Path to the image file */
+  imagePath: string;
+  /** Point coordinates as [x, y] pairs */
+  points: [number, number][];
+  /** Point labels: 1 for foreground, 0 for background */
+  pointLabels: number[];
+  /** Optional low-res mask from previous prediction for refinement */
+  maskInput?: number[][];
+  /** Whether to return multiple mask options */
+  multimaskOutput?: boolean;
+}
+
+export interface SegmentationPredictResponse {
+  /** Whether the prediction succeeded */
+  success: boolean;
+  /** Error message if failed */
+  error?: string;
+  /** Polygon coordinates as [x, y] pairs */
+  polygon?: [number, number][];
+  /** Bounding box [x_min, y_min, x_max, y_max] */
+  bounds?: [number, number, number, number];
+  /** Quality score from segmentation model */
+  score?: number;
+  /** Low-res mask for subsequent refinement */
+  lowResMask?: number[][];
+  /** Mask dimensions [height, width] */
+  maskShape?: [number, number];
+  /** RLE-encoded full-resolution mask for display: [[value, count], ...] */
+  rleMask?: [number, number][];
+}
+
+/**
+ * Stereo point-segmentation. The segmentation service warps the seed to the
+ * other camera (configured stereo backend), segments there, and -- when enabled
+ * -- derives head/tail lines + the measurement.
+ */
+export interface SegmentationStereoSegmentRequest {
+  /** The already-segmented source-camera polygon (sampling + measurement). */
+  polygon?: [number, number][];
+  /** Source-camera click points and labels. */
+  points: [number, number][];
+  pointLabels: number[];
+  /** Source (clicked) and other camera image/video paths. */
+  sourceImagePath: string;
+  otherImagePath: string;
+  /** Calibration file path, read by the embedded stereo warper. */
+  calibrationFile?: string;
+  /** Time in seconds when the paths are video files. */
+  frameTime?: number;
+}
+
+export interface SegmentationStereoSegmentResponse {
+  success: boolean;
+  error?: string;
+  /** Other-camera polygon from SAM. */
+  polygon?: [number, number][];
+  bounds?: [number, number, number, number];
+  score?: number;
+  /** Seed point(s) used on the other camera (median of warped samples). */
+  seedPoints?: [number, number][];
+  seedLabels?: number[];
+  /** Optional head/tail lines: source = clicked camera, other = warped. */
+  generateLine?: boolean;
+  lineSource?: [[number, number], [number, number]];
+  lineOther?: [[number, number], [number, number]];
+  /** Stereo measurement for the derived line (calibration units, e.g. mm). */
+  measurement?: {
+    length: number;
+    midpoint_x: number;
+    midpoint_y: number;
+    midpoint_z: number;
+    midpoint_range: number;
+    stereo_rms: number;
+  };
+}
+
+export interface SegmentationStatusResponse {
+  /** Whether segmentation is available */
+  available: boolean;
+  /** Whether the model is currently loaded */
+  loaded?: boolean;
+  /** Whether the service is ready for predictions */
+  ready?: boolean;
+}
+
 export {
   provideApi,
   useApi,

client/dive-common/components/DeleteControls.vue

@@ -24,8 +24,20 @@ export default Vue.extend({
       if (this.editingMode === 'rectangle') {
         return true; // deleting rectangle is unsupported
       }
+      if (this.editingMode === 'Point') {
+        return true; // Point mode uses reset instead of delete
+      }
       return false;
     },
+    isPolygonMode(): boolean {
+      return this.editingMode === 'Polygon';
+    },
+    editModeIcon(): string {
+      if (this.editingMode === 'Polygon') return 'mdi-vector-polygon';
+      if (this.editingMode === 'LineString') return 'mdi-vector-line';
+      if (this.editingMode === 'rectangle') return 'mdi-vector-square';
+      return 'mdi-shape';
+    },
   },
 
   methods: {
@@ -39,33 +51,104 @@ export default Vue.extend({
         this.$emit('delete-annotation');
       }
     },
+    addHole() {
+      this.$emit('add-hole');
+    },
+    addPolygon() {
+      this.$emit('add-polygon');
+    },
   },
 });
 </script>
 
 <template>
-  <span class="mx-1">
-    <v-btn
+  <span class="mx-1 d-flex align-center">
+    <!-- Add Polygon button - shown in polygon edit mode -->
+    <v-tooltip
+      v-if="isPolygonMode"
+      bottom
+    >
+      <template #activator="{ on, attrs }">
+        <v-btn
+          v-bind="attrs"
+          color="primary"
+          depressed
+          small
+          class="mr-1"
+          v-on="on"
+          @click="addPolygon"
+        >
+          <v-icon small>
+            mdi-vector-polygon
+          </v-icon>
+          <v-icon
+            x-small
+            class="ml-n1"
+          >
+            mdi-plus-circle-outline
+          </v-icon>
+        </v-btn>
+      </template>
+      <span>Add another polygon</span>
+    </v-tooltip>
+
+    <!-- Add Hole button - shown in polygon edit mode -->
+    <v-tooltip
+      v-if="isPolygonMode"
+      bottom
+    >
+      <template #activator="{ on, attrs }">
+        <v-btn
+          v-bind="attrs"
+          color="primary"
+          depressed
+          small
+          class="mr-1"
+          v-on="on"
+          @click="addHole"
+        >
+          <v-icon small>
+            mdi-vector-polygon
+          </v-icon>
+          <v-icon
+            x-small
+            class="ml-n1"
+          >
+            mdi-minus-circle-outline
+          </v-icon>
+        </v-btn>
+      </template>
+      <span>Add hole to polygon</span>
+    </v-tooltip>
+
+    <!-- Delete button -->
+    <v-tooltip
       v-if="!disabled"
-      color="error"
-      depressed
-      small
-      @click="deleteSelected"
+      bottom
     >
-      <pre class="mr-1 text-body-2">del</pre>
-      <span v-if="selectedFeatureHandle >= 0">
-        point {{ selectedFeatureHandle }}
-      </span>
-      <span v-else-if="editingMode">
-        {{ editingMode }}
-      </span>
-      <span v-else>unselected</span>
-      <v-icon
-        small
-        class="ml-2"
-      >
-        mdi-delete
-      </v-icon>
-    </v-btn>
+      <template #activator="{ on, attrs }">
+        <v-btn
+          v-bind="attrs"
+          color="error"
+          depressed
+          small
+          v-on="on"
+          @click="deleteSelected"
+        >
+          <span class="mr-1 text-body-2 font-weight-bold">DEL</span>
+          <span v-if="selectedFeatureHandle >= 0">
+            pt{{ selectedFeatureHandle }}
+          </span>
+          <v-icon
+            v-else
+            small
+          >
+            {{ editModeIcon }}
+          </v-icon>
+        </v-btn>
+      </template>
+      <span v-if="selectedFeatureHandle >= 0">Delete point {{ selectedFeatureHandle }}</span>
+      <span v-else-if="editingMode">Delete {{ editingMode }}</span>
+    </v-tooltip>
   </span>
 </template>

client/dive-common/components/EditorMenu.vue

@@ -11,6 +11,7 @@ import { flatten } from 'lodash';
 import { Mousetrap } from 'vue-media-annotator/types';
 import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers';
 import Recipe from 'vue-media-annotator/recipe';
+import SegmentationPointClick from 'dive-common/recipes/segmentationpointclick';
 
 import AnnotationVisibilityMenu from './AnnotationVisibilityMenu.vue';
 
@@ -19,6 +20,7 @@ interface ButtonData {
   icon: string;
   type?: VisibleAnnotationTypes;
   active: boolean;
+  loading?: boolean;
   mousetrap?: Mousetrap[];
   description: string;
   click: () => void;
@@ -75,7 +77,11 @@ export default defineComponent({
       default: true,
     },
   },
-  emits: ['set-annotation-state', 'update:tail-settings', 'update:show-user-created-icon'],
+  emits: [
+    'set-annotation-state',
+    'update:tail-settings',
+    'update:show-user-created-icon',
+  ],
   setup(props, { emit }) {
     const toolTimeTimeout = ref<number | null>(null);
     const STORAGE_KEY = 'editorMenu.editButtonsExpanded';
@@ -129,6 +135,7 @@ export default defineComponent({
           id: r.name,
           icon: r.icon.value || 'mdi-pencil',
           active: props.editingTrack && r.active.value,
+          loading: r.loading?.value ?? false,
           description: r.name,
           click: () => r.activate(),
           mousetrap: [
@@ -142,7 +149,9 @@ export default defineComponent({
       ];
     });
 
-    const mousetrap = computed((): Mousetrap[] => flatten(editButtons.value.map((b) => b.mousetrap || [])));
+    const mousetrap = computed((): Mousetrap[] => [
+      ...flatten(editButtons.value.map((b) => b.mousetrap || [])),
+    ]);
 
     const activeEditButton = computed(() => editButtons.value.find((b) => b.active) || editButtons.value[0]);
 
@@ -175,6 +184,13 @@ export default defineComponent({
       return { text: 'Not editing', icon: 'mdi-pencil-off-outline', color: '' };
     });
 
+    const activeSegmentationRecipe = computed((): SegmentationPointClick | null => {
+      const segRecipe = props.recipes.find(
+        (r) => r instanceof SegmentationPointClick && r.active.value,
+      ) as SegmentationPointClick | undefined;
+      return segRecipe || null;
+    });
+
     const editingTooltip = computed(() => {
       if (props.editingDetails === 'disabled' || !props.editingMode || typeof props.editingMode !== 'string') {
         return '';
@@ -208,6 +224,7 @@ export default defineComponent({
       toggleEditButtonsExpanded,
       activeEditButton,
       editButtonsMenuKey,
+      activeSegmentationRecipe,
     };
   },
 });
@@ -265,15 +282,17 @@ export default defineComponent({
         <template #activator="{ on, attrs }">
           <v-btn
             v-bind="attrs"
-            :disabled="!editingMode"
+            :disabled="!editingMode || activeEditButton?.loading"
             :outlined="!activeEditButton?.active"
             :color="activeEditButton?.active ? editingHeader.color : ''"
             class="mx-1"
             small
             v-on="on"
           >
             <pre v-if="activeEditButton?.mousetrap">{{ activeEditButton.mousetrap[0].bind }}:</pre>
-            <v-icon>{{ activeEditButton?.icon }}</v-icon>
+            <v-icon :class="{ 'mdi-spin': activeEditButton?.loading }">
+              {{ activeEditButton?.icon }}
+            </v-icon>
             <v-btn
               icon
               x-small
@@ -293,15 +312,17 @@ export default defineComponent({
           >
             <v-list-item-icon>
               <v-btn
-                :disabled="!editingMode"
+                :disabled="!editingMode || button.loading"
                 :outlined="!button.active"
                 :color="button.active ? editingHeader.color : ''"
                 class="mx-1"
                 small
                 @click="button.click"
               >
                 <pre v-if="button.mousetrap">{{ button.mousetrap[0].bind }}:</pre>
-                <v-icon>{{ button.icon }}</v-icon>
+                <v-icon :class="{ 'mdi-spin': button.loading }">
+                  {{ button.icon }}
+                </v-icon>
               </v-btn>
             </v-list-item-icon>
             <v-list-item-content>
@@ -332,18 +353,43 @@ export default defineComponent({
         <v-btn
           v-for="button in editButtons"
           :key="button.id + 'view'"
-          :disabled="!editingMode"
+          :disabled="!editingMode || button.loading"
           :outlined="!button.active"
           :color="button.active ? editingHeader.color : ''"
           class="mx-1"
           small
           @click="button.click"
         >
           <pre v-if="button.mousetrap">{{ button.mousetrap[0].bind }}:</pre>
-          <v-icon>{{ button.icon }}</v-icon>
+          <v-icon :class="{ 'mdi-spin': button.loading }">
+            {{ button.icon }}
+          </v-icon>
         </v-btn>
       </template>
-      <slot name="delete-controls" />
+      <!-- Segmentation Reset button -->
+      <template v-if="activeSegmentationRecipe && editingMode === 'Point'">
+        <v-divider
+          vertical
+          class="mx-2"
+        />
+        <v-btn
+          color="error"
+          class="mx-1"
+          small
+          :disabled="!activeSegmentationRecipe.hasPoints()"
+          @click="activeSegmentationRecipe.resetPoints()"
+        >
+          <v-icon left>
+            mdi-close
+          </v-icon>
+          Reset
+        </v-btn>
+      </template>
+      <!-- Hide delete controls when in segmentation mode -->
+      <slot
+        v-if="!activeSegmentationRecipe"
+        name="delete-controls"
+      />
       <slot name="multicam-controls-left" />
       <v-spacer />
       <slot name="multicam-controls-right" />