diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 55fca437c..78c40b8ec 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -418,6 +418,81 @@ export interface SegmentationStatusResponse { ready?: boolean; } +/** + * Text Query Types for open-vocabulary detection/segmentation + */ + +/** A single detection returned from a text query */ +export interface TextQueryDetection { + /** Bounding box [x1, y1, x2, y2] */ + box: [number, number, number, number]; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Confidence score */ + score: number; + /** Label/class name (often the query text) */ + label: string; + /** Low-res mask for refinement (optional) */ + lowResMask?: number[][]; +} + +export interface TextQueryRequest { + /** Path to the image file (or video file for video datasets) */ + imagePath: string; + /** Frame time in seconds, required when imagePath is a video so the service + * extracts the correct frame. Omitted for image-sequence datasets. */ + frameTime?: number; + /** Text query describing what to find (e.g., "fish", "person swimming") */ + text: string; + /** Confidence threshold for detections (default: 0.3) */ + boxThreshold?: number; + /** Maximum number of detections to return (default: 10) */ + maxDetections?: number; + /** Optional boxes to refine [x1, y1, x2, y2][] */ + boxes?: [number, number, number, number][]; + /** Optional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Optional masks to refine */ + masks?: number[][][]; +} + +export interface TextQueryResponse { + /** Whether the query succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** List of detections found */ + detections?: TextQueryDetection[]; + /** The original query text */ + query?: string; + /** Whether fallback method was used (no native text support) */ + fallback?: boolean; +} + +export interface RefineDetectionsRequest { + /** Path to the image file */ + imagePath: string; + /** Detections to refine */ + detections: TextQueryDetection[]; + /** Optional additional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for additional points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Whether to include refined masks in response */ + refineMasks?: boolean; +} + +export interface RefineDetectionsResponse { + /** Whether the refinement succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Refined detections */ + detections?: TextQueryDetection[]; +} + export { provideApi, useApi, diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 4366737c6..8073bcd46 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -23,11 +23,15 @@ interface ButtonData { type?: VisibleAnnotationTypes; active: boolean; loading?: boolean; + unavailable?: boolean; + unavailableTooltip?: string; mousetrap?: Mousetrap[]; description: string; click: () => void; } +const SAM3_ADDON_WIKI_URL = 'https://github.com/VIAME/VIAME/wiki/Model-Zoo-and-Add-Ons'; + export default defineComponent({ name: 'EditorMenu', components: { @@ -80,11 +84,23 @@ export default defineComponent({ type: Boolean, default: true, }, + textQueryEnabled: { + type: Boolean, + default: false, + }, + textQueryAvailable: { + type: Boolean, + default: false, + }, }, emits: [ 'set-annotation-state', 'update:tail-settings', 'update:show-user-created-icon', + 'text-query-init', + 'text-query', + 'text-query-all-frames', + 'open-external-link', ], setup(props, { emit }) { const toolTimeTimeout = ref(null); @@ -103,6 +119,87 @@ export default defineComponent({ localStorage.setItem(STORAGE_KEY, String(value)); }); + // Text query state + const textQueryDialogOpen = ref(false); + const textQueryInput = ref(''); + const textQueryLoading = ref(false); + const textQueryThreshold = ref(0.3); + const textQueryInitializing = ref(false); + const textQueryServiceError = ref(''); + const textQueryAllFrames = ref(false); + // When on, existing annotations are removed before the query results are + // applied. On by default so a query replaces rather than accumulates. + const textQueryReplaceExisting = ref(true); + const sam3InfoDialogOpen = ref(false); + + const openSam3InfoDialog = () => { + sam3InfoDialogOpen.value = true; + }; + + const closeSam3InfoDialog = () => { + sam3InfoDialogOpen.value = false; + }; + + const openSam3AddonWiki = () => { + emit('open-external-link', SAM3_ADDON_WIKI_URL); + }; + + const handleTextQueryClick = () => { + if (!props.textQueryAvailable) { + openSam3InfoDialog(); + return; + } + openTextQueryDialog(); + }; + + const openTextQueryDialog = () => { + textQueryDialogOpen.value = true; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryAllFrames.value = false; + textQueryReplaceExisting.value = true; + textQueryInitializing.value = true; + emit('text-query-init'); + }; + + const closeTextQueryDialog = () => { + textQueryDialogOpen.value = false; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryInitializing.value = false; + textQueryAllFrames.value = false; + textQueryReplaceExisting.value = true; + }; + + const onTextQueryServiceReady = (success: boolean, error?: string) => { + textQueryInitializing.value = false; + if (!success) { + textQueryServiceError.value = error || 'Text query service is not available'; + } + }; + + const submitTextQuery = () => { + if (!textQueryInput.value.trim()) { + return; + } + textQueryLoading.value = true; + if (textQueryAllFrames.value) { + emit('text-query-all-frames', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + replaceExisting: textQueryReplaceExisting.value, + }); + } else { + emit('text-query', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + replaceExisting: textQueryReplaceExisting.value, + }); + } + closeTextQueryDialog(); + textQueryLoading.value = false; + }; + const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -151,6 +248,20 @@ export default defineComponent({ ...r.mousetrap(), ], })), + /* Text Query button included alongside other annotation types (desktop only) */ + ...(props.textQueryEnabled ? [{ + id: 'Text Query', + icon: 'mdi-text-search', + active: false, + unavailable: !props.textQueryAvailable, + unavailableTooltip: 'SAM3 add-on not installed. Click for more information.', + description: 'Text Query', + mousetrap: [{ + bind: 'q', + handler: () => handleTextQueryClick(), + }], + click: () => handleTextQueryClick(), + }] : []), ]; }); @@ -253,6 +364,22 @@ export default defineComponent({ segmentationPredicting, segmentationLoading, segmentationTooltip, + // Text query + textQueryDialogOpen, + textQueryInput, + textQueryLoading, + textQueryThreshold, + textQueryInitializing, + textQueryServiceError, + textQueryAllFrames, + textQueryReplaceExisting, + openTextQueryDialog, + closeTextQueryDialog, + onTextQueryServiceReady, + submitTextQuery, + sam3InfoDialogOpen, + closeSam3InfoDialog, + openSam3AddonWiki, }; }, }); @@ -342,19 +469,33 @@ export default defineComponent({ :key="`${button.id}-menu`" > - -
{{ button.mousetrap[0].bind }}:
- - {{ button.icon }} - -
+ + {{ button.unavailableTooltip }} +
{{ button.id }} @@ -380,21 +521,36 @@ export default defineComponent({ /> - -
{{ button.mousetrap[0].bind }}:
- - {{ button.icon }} - -
+ + {{ button.unavailableTooltip }} + @@ -434,6 +590,163 @@ export default defineComponent({ @update:show-user-created-icon="$emit('update:show-user-created-icon', $event)" /> + + + + + + + mdi-text-search + + Text Query + + + +
+ +

+ Loading text query model... +

+
+ +
+ + mdi-alert-circle + +

+ {{ textQueryServiceError }} +

+
+ + +

+ Textual query support uses architectures derived from Meta's SAM3 project +

+
+ + + + {{ textQueryServiceError ? 'Close' : 'Cancel' }} + + + Search + + +
+
+ + + + + + + mdi-package-down + + SAM3 Add-On Required + + +

+ Text query requires the SAM3 Text Query Segmentation and Tracking Models + add-on to be installed in your VIAME directory. +

+

+ You can download the add-on from the + + VIAME Model Zoo and Add-Ons + + page. Extract the package and merge its folders into your existing VIAME + installation. +

+
+ + + + Close + + + Open Model Zoo + + +
+
@@ -444,8 +757,18 @@ export default defineComponent({ border-radius: 16px; white-space: nowrap; border: 1px solid; - cursor: default; } + +.edit-btn-unavailable { + opacity: 0.45 !important; +} + +.sam3-wiki-link { + color: var(--v-primary-base); + cursor: pointer; + text-decoration: underline; +} + .mode-button{ border: 1px solid grey; min-width: 36px; diff --git a/client/dive-common/components/UserGuideDialog.vue b/client/dive-common/components/UserGuideDialog.vue index 606cc2939..ba6480db5 100644 --- a/client/dive-common/components/UserGuideDialog.vue +++ b/client/dive-common/components/UserGuideDialog.vue @@ -1,8 +1,16 @@