From 25ad49b40f3430f427284d3c2662b66378b12e3e Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Mon, 29 Jun 2026 20:41:25 -0400 Subject: [PATCH 01/13] Add interactive text-query annotation button SAM3 text-query button, dialog, API, and IPC handlers, on top of the merged interactive segmentation + stereo work on main. The button lives in the annotation type list (with the 't' shortcut) alongside rectangle/polygon/line/segment, and opens a dialog to run a SAM3 text query on the current frame or, optionally, across all frames as a pipeline job. Desktop-only: gated behind a textQueryEnabled prop set only by the desktop ViewerLoader, so the web client renders nothing. Brings the feature to parity with origin/viame/master (SAM3 attribution text, persistent dialog while loading, exit edit mode on completion). --- client/dive-common/apispec.ts | 72 ++++++ client/dive-common/components/EditorMenu.vue | 186 ++++++++++++++ client/dive-common/components/Viewer.vue | 22 ++ client/platform/desktop/backend/ipcService.ts | 43 ++++ .../desktop/backend/native/interactive.ts | 47 ++++ client/platform/desktop/frontend/api.ts | 49 ++++ .../frontend/components/ViewerLoader.vue | 226 +++++++++++++++++- 7 files changed, 643 insertions(+), 2 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 55fca437c..763cde7e9 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -418,6 +418,78 @@ export interface SegmentationStatusResponse { ready?: boolean; } +/** + * Text Query Types for open-vocabulary detection/segmentation + */ + +/** A single detection returned from a text query */ +export interface TextQueryDetection { + /** Bounding box [x1, y1, x2, y2] */ + box: [number, number, number, number]; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Confidence score */ + score: number; + /** Label/class name (often the query text) */ + label: string; + /** Low-res mask for refinement (optional) */ + lowResMask?: number[][]; +} + +export interface TextQueryRequest { + /** Path to the image file */ + imagePath: string; + /** Text query describing what to find (e.g., "fish", "person swimming") */ + text: string; + /** Confidence threshold for detections (default: 0.3) */ + boxThreshold?: number; + /** Maximum number of detections to return (default: 10) */ + maxDetections?: number; + /** Optional boxes to refine [x1, y1, x2, y2][] */ + boxes?: [number, number, number, number][]; + /** Optional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Optional masks to refine */ + masks?: number[][][]; +} + +export interface TextQueryResponse { + /** Whether the query succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** List of detections found */ + detections?: TextQueryDetection[]; + /** The original query text */ + query?: string; + /** Whether fallback method was used (no native text support) */ + fallback?: boolean; +} + +export interface RefineDetectionsRequest { + /** Path to the image file */ + imagePath: string; + /** Detections to refine */ + detections: TextQueryDetection[]; + /** Optional additional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for additional points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Whether to include refined masks in response */ + refineMasks?: boolean; +} + +export interface RefineDetectionsResponse { + /** Whether the refinement succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Refined detections */ + detections?: TextQueryDetection[]; +} + export { provideApi, useApi, diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 4366737c6..761c189e7 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -80,11 +80,18 @@ export default defineComponent({ type: Boolean, default: true, }, + textQueryEnabled: { + type: Boolean, + default: false, + }, }, emits: [ 'set-annotation-state', 'update:tail-settings', 'update:show-user-created-icon', + 'text-query-init', + 'text-query', + 'text-query-all-frames', ], setup(props, { emit }) { const toolTimeTimeout = ref(null); @@ -103,6 +110,59 @@ export default defineComponent({ localStorage.setItem(STORAGE_KEY, String(value)); }); + // Text query state + const textQueryDialogOpen = ref(false); + const textQueryInput = ref(''); + const textQueryLoading = ref(false); + const textQueryThreshold = ref(0.3); + const textQueryInitializing = ref(false); + const textQueryServiceError = ref(''); + const textQueryAllFrames = ref(false); + + const openTextQueryDialog = () => { + textQueryDialogOpen.value = true; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryAllFrames.value = false; + textQueryInitializing.value = true; + emit('text-query-init'); + }; + + const closeTextQueryDialog = () => { + textQueryDialogOpen.value = false; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryInitializing.value = false; + textQueryAllFrames.value = false; + }; + + const onTextQueryServiceReady = (success: boolean, error?: string) => { + textQueryInitializing.value = false; + if (!success) { + textQueryServiceError.value = error || 'Text query service is not available'; + } + }; + + const submitTextQuery = () => { + if (!textQueryInput.value.trim()) { + return; + } + textQueryLoading.value = true; + if (textQueryAllFrames.value) { + emit('text-query-all-frames', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } else { + emit('text-query', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } + closeTextQueryDialog(); + textQueryLoading.value = false; + }; + const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -151,6 +211,18 @@ export default defineComponent({ ...r.mousetrap(), ], })), + /* Text Query button included alongside other annotation types (desktop only) */ + ...(props.textQueryEnabled ? [{ + id: 'Text Query', + icon: 'mdi-text-search', + active: false, + description: 'Text Query', + mousetrap: [{ + bind: 't', + handler: () => openTextQueryDialog(), + }], + click: () => openTextQueryDialog(), + }] : []), ]; }); @@ -253,6 +325,18 @@ export default defineComponent({ segmentationPredicting, segmentationLoading, segmentationTooltip, + // Text query + textQueryDialogOpen, + textQueryInput, + textQueryLoading, + textQueryThreshold, + textQueryInitializing, + textQueryServiceError, + textQueryAllFrames, + openTextQueryDialog, + closeTextQueryDialog, + onTextQueryServiceReady, + submitTextQuery, }; }, }); @@ -434,6 +518,108 @@ export default defineComponent({ @update:show-user-created-icon="$emit('update:show-user-created-icon', $event)" /> + + + + + + + mdi-text-search + + Text Query + + + +
+ +

+ Loading text query model... +

+
+ +
+ + mdi-alert-circle + +

+ {{ textQueryServiceError }} +

+
+ + +

+ Textual query support uses architectures derived from Meta's SAM3 project +

+ + + + + {{ textQueryServiceError ? 'Close' : 'Cancel' }} + + + Search + + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index ea2580422..7bd5d57bd 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -127,6 +127,10 @@ export default defineComponent({ type: Array as PropType, default: () => [], }, + textQueryEnabled: { + type: Boolean, + default: false, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -170,6 +174,17 @@ export default defineComponent({ const controlsRef = ref(); const controlsHeight = ref(0); const controlsCollapsed = ref(false); + const editorMenuRef = ref(); + + /** + * Forward text query service ready status to EditorMenu + * Called by ViewerLoader when text query service initialization completes + */ + function onTextQueryServiceReady(success: boolean, error?: string) { + if (editorMenuRef.value?.onTextQueryServiceReady) { + editorMenuRef.value.onTextQueryServiceReady(success, error); + } + } const sideBarCollapsed = ref(false); // Sidebar mode: 'left', 'bottom', or 'collapsed' @@ -1233,6 +1248,8 @@ export default defineComponent({ controlsHeight, controlsCollapsed, sideBarCollapsed, + editorMenuRef, + onTextQueryServiceReady, sidebarMode, cycleSidebarMode, sidebarModeIcon, @@ -1417,6 +1434,7 @@ export default defineComponent({ + + + + Text Query + + +
+ Searching the current frame… +
+ + + + Date: Wed, 1 Jul 2026 13:39:53 -0400 Subject: [PATCH 05/13] Apply text-query detection confidence to created tracks Single-frame text query created every track with the default confidence of 1.0 because TrackStore.add() seeds the confidence pair at 1.0 and the handler never applied the detection's score. Call setType with det.score so the model's confidence is reflected instead of always showing 100%. --- .../platform/desktop/frontend/components/ViewerLoader.vue | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/platform/desktop/frontend/components/ViewerLoader.vue b/client/platform/desktop/frontend/components/ViewerLoader.vue index cff5df0cf..bfac391db 100644 --- a/client/platform/desktop/frontend/components/ViewerLoader.vue +++ b/client/platform/desktop/frontend/components/ViewerLoader.vue @@ -437,6 +437,14 @@ export default defineComponent({ newTrackId, // Use the new track ID ); + // add() seeds the confidence pair at 1.0; apply the detection's + // actual confidence from the text-query model so it isn't shown + // as 100%. (setType clamps >=1 to 1.0, which is the correct + // behavior for a genuinely-1.0 score.) + if (typeof det.score === 'number') { + newTrack.setType(det.label, det.score); + } + // Calculate bounds from box [x1, y1, x2, y2] const [x1, y1, x2, y2] = det.box; const bounds = [x1, y1, x2, y2] as [number, number, number, number]; From 021d883059d80300b9b2439bd759fb670407550f Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 1 Jul 2026 15:18:38 -0400 Subject: [PATCH 06/13] Surface pipeline job failures to the user When a job process exits non-zero (and was not cancelled), scan its captured output for lines beginning with 'ERROR:' and show them in a dialog. Processes emit a single 'ERROR: ' line to report the real failure reason (e.g. a GPU out-of-memory during SAM3 model load), so the user sees the cause without opening the job log. --- .../platform/desktop/frontend/store/jobs.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/client/platform/desktop/frontend/store/jobs.ts b/client/platform/desktop/frontend/store/jobs.ts index db1b1f735..e2a8e3f43 100644 --- a/client/platform/desktop/frontend/store/jobs.ts +++ b/client/platform/desktop/frontend/store/jobs.ts @@ -16,6 +16,7 @@ import { RunTraining, JsonMeta, } from 'platform/desktop/constants'; +import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import AsyncGpuJobQueue from './queues/asyncGpuJobQueue'; import AsyncCpuJobQueue from './queues/asyncCpuJobQueue'; import { setRecents } from './dataset'; @@ -24,6 +25,9 @@ interface DesktopJobHistory { job: DesktopJob; truncatedLogs: string[]; totalLogLength: number; + // Set once we've surfaced this job's failure to the user, so the repeated + // updates that arrive don't re-open the dialog. + errorNotified?: boolean; } const truncateOutputAtLines = 500; @@ -64,6 +68,36 @@ export function updateHistory(args: DesktopJobUpdate) { if (args.endTime !== undefined) { existing.job.endTime = args.endTime; } + + // Surface a failed job's error to the user. The process reports the cause on + // lines beginning with "ERROR:" (DIVE convention); we only prompt when the + // job has actually exited with a non-zero, non-cancellation code and at + // least one such line was captured. + const finished = existing.job.endTime !== undefined; + const failed = existing.job.exitCode !== null + && existing.job.exitCode !== 0 + && existing.job.exitCode !== cancelledJobExitCode + && !existing.job.cancelledJob; + if (finished && failed && !existing.errorNotified) { + existing.errorNotified = true; + const errorLines = existing.truncatedLogs + .map((line) => line.trim()) + .filter((line) => line.startsWith('ERROR:')) + .map((line) => line.replace(/^ERROR:\s*/, '')); + if (errorLines.length > 0) { + try { + const { prompt } = usePrompt(); + prompt({ + title: `${existing.job.title || 'Job'} failed`, + text: errorLines, + positiveButton: 'OK', + }); + } catch { + // Prompt service not available (e.g. very early startup); the error + // is still visible in the job log. + } + } + } } const conversionJob: Ref> = ref({}); From c96182cc190b80532a0ac4e5353f9d6623cb6c38 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Wed, 1 Jul 2026 15:31:37 -0400 Subject: [PATCH 07/13] add warning and model zoo links if not installed --- client/dive-common/components/EditorMenu.vue | 176 +++++++++++++++--- client/dive-common/components/Viewer.vue | 6 + client/platform/desktop/backend/ipcService.ts | 19 +- client/platform/desktop/frontend/api.ts | 5 + .../frontend/components/ViewerLoader.vue | 21 +++ 5 files changed, 199 insertions(+), 28 deletions(-) diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index c15e3621e..1dc4a5495 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -23,11 +23,15 @@ interface ButtonData { type?: VisibleAnnotationTypes; active: boolean; loading?: boolean; + unavailable?: boolean; + unavailableTooltip?: string; mousetrap?: Mousetrap[]; description: string; click: () => void; } +const SAM3_ADDON_WIKI_URL = 'https://github.com/VIAME/VIAME/wiki/Model-Zoo-and-Add-Ons'; + export default defineComponent({ name: 'EditorMenu', components: { @@ -84,6 +88,10 @@ export default defineComponent({ type: Boolean, default: false, }, + textQueryAvailable: { + type: Boolean, + default: false, + }, }, emits: [ 'set-annotation-state', @@ -92,6 +100,7 @@ export default defineComponent({ 'text-query-init', 'text-query', 'text-query-all-frames', + 'open-external-link', ], setup(props, { emit }) { const toolTimeTimeout = ref(null); @@ -121,6 +130,27 @@ export default defineComponent({ // When on, existing annotations are removed before the query results are // applied. Off by default so results are added alongside existing ones. const textQueryReplaceExisting = ref(false); + const sam3InfoDialogOpen = ref(false); + + const openSam3InfoDialog = () => { + sam3InfoDialogOpen.value = true; + }; + + const closeSam3InfoDialog = () => { + sam3InfoDialogOpen.value = false; + }; + + const openSam3AddonWiki = () => { + emit('open-external-link', SAM3_ADDON_WIKI_URL); + }; + + const handleTextQueryClick = () => { + if (!props.textQueryAvailable) { + openSam3InfoDialog(); + return; + } + openTextQueryDialog(); + }; const openTextQueryDialog = () => { textQueryDialogOpen.value = true; @@ -223,12 +253,14 @@ export default defineComponent({ id: 'Text Query', icon: 'mdi-text-search', active: false, + unavailable: !props.textQueryAvailable, + unavailableTooltip: 'SAM3 add-on not installed. Click for more information.', description: 'Text Query', mousetrap: [{ bind: 't', - handler: () => openTextQueryDialog(), + handler: () => handleTextQueryClick(), }], - click: () => openTextQueryDialog(), + click: () => handleTextQueryClick(), }] : []), ]; }); @@ -345,6 +377,9 @@ export default defineComponent({ closeTextQueryDialog, onTextQueryServiceReady, submitTextQuery, + sam3InfoDialogOpen, + closeSam3InfoDialog, + openSam3AddonWiki, }; }, }); @@ -434,19 +469,33 @@ export default defineComponent({ :key="`${button.id}-menu`" > - - 
{{ button.mousetrap[0].bind }}:
- - {{ button.icon }} - - + + {{ button.unavailableTooltip }} + {{ button.id }} @@ -472,21 +521,36 @@ export default defineComponent({ /> - - 
{{ button.mousetrap[0].bind }}:
- - {{ button.icon }} - - + + {{ button.unavailableTooltip }} + @@ -635,6 +699,54 @@ export default defineComponent({ + + + + + + + mdi-package-down + + SAM3 Add-On Required + + +

+ Text query requires the SAM3 Text Query Segmentation and Tracking Models + add-on to be installed in your VIAME directory. +

+

+ You can download the add-on from the + + VIAME Model Zoo and Add-Ons + + page. Extract the package and merge its folders into your existing VIAME + installation. +

+ + + + + Close + + + Open Model Zoo + + + + @@ -645,8 +757,18 @@ export default defineComponent({ border-radius: 16px; white-space: nowrap; border: 1px solid; - cursor: default; } + +.edit-btn-unavailable { + opacity: 0.45 !important; +} + +.sam3-wiki-link { + color: var(--v-primary-base); + cursor: pointer; + text-decoration: underline; +} + .mode-button{ border: 1px solid grey; min-width: 36px; diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 02ac539e2..5f8784660 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -131,6 +131,10 @@ export default defineComponent({ type: Boolean, default: false, }, + textQueryAvailable: { + type: Boolean, + default: false, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -1460,6 +1464,7 @@ export default defineComponent({ lassoModeActive: !readonlyState && lassoModeActive, lassoDrawing: !readonlyState && lassoDrawing, textQueryEnabled, + textQueryAvailable, }" :tail-settings.sync="clientSettings.annotatorPreferences.trackTails" :show-user-created-icon.sync="clientSettings.annotatorPreferences.showUserCreatedIcon" @@ -1468,6 +1473,7 @@ export default defineComponent({ @text-query-init="$emit('text-query-init')" @text-query="onTextQuerySubmit" @text-query-all-frames="$emit('text-query-all-frames', $event)" + @open-external-link="$emit('open-external-link', $event)" >