feat: auto-suggest ISR missions from video preview frame

On video select, extract a frame client-side and send to GPT-4o-mini
vision to suggest 3-5 feasible mission objectives. Suggestions render
as clickable chips below the mission input — click to populate.

- New BAML SuggestMissions function with detailed vision prompt
constraining suggestions to COCO-detectable, analyst-decidable missions
- New POST /suggest-missions endpoint
- Frontend frame extraction with race guard, video/canvas cleanup,
and 512px cap for efficient LLM processing
- Suggestion chips with fade-in animation, hover tooltips, selection state

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

app.py

@@ -27,6 +27,7 @@ except Exception as e:
     print(f"Startup Diagnostics Error: {e}")
 
 import asyncio
+import base64
 import json
 import shutil
 import tempfile
@@ -399,6 +400,25 @@ async def detect_endpoint(
     return response
 
 
+@app.post("/suggest-missions")
+async def suggest_missions_endpoint(frame: UploadFile = File(...)):
+    """Analyze a video frame and suggest 3-5 ISR mission objectives."""
+    try:
+        from baml_client.sync_client import b as baml
+        from baml_py import Image
+
+        data = await frame.read()
+        b64 = base64.b64encode(data).decode("utf-8")
+        media_type = frame.content_type or "image/jpeg"
+        frame_image = Image.from_base64(media_type, b64)
+
+        suggestions = await asyncio.to_thread(baml.SuggestMissions, frame=frame_image)
+        return [{"mission": s.mission, "reasoning": s.reasoning} for s in suggestions]
+    except Exception:
+        logging.exception("SuggestMissions failed")
+        return []
+
+
 @app.post("/detect/async")
 async def detect_async_endpoint(
     video: UploadFile = File(...),

baml_client/async_client.py

@@ -112,6 +112,21 @@ class BamlAsyncClient:
                 "mission_text": mission_text,
             })
             return typing.cast(types.MissionPlan, __result__.cast_to(types, types, stream_types, False, __runtime__))
+    async def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> typing.List["types.MissionSuggestion"]:
+        # Check if on_tick is provided
+        if 'on_tick' in baml_options:
+            # Use streaming internally when on_tick is provided
+            __stream__ = self.stream.SuggestMissions(frame=frame,
+                baml_options=baml_options)
+            return await __stream__.get_final_response()
+        else:
+            # Original non-streaming code
+            __result__ = await self.__options.merge_options(baml_options).call_function_async(function_name="SuggestMissions", args={
+                "frame": frame,
+            })
+            return typing.cast(typing.List["types.MissionSuggestion"], __result__.cast_to(types, types, stream_types, False, __runtime__))
     
 
 
@@ -145,6 +160,18 @@ class BamlStreamClient:
           lambda x: typing.cast(types.MissionPlan, x.cast_to(types, types, stream_types, False, __runtime__)),
           __ctx__,
         )
+    def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.BamlStream[typing.List["stream_types.MissionSuggestion"], typing.List["types.MissionSuggestion"]]:
+        __ctx__, __result__ = self.__options.merge_options(baml_options).create_async_stream(function_name="SuggestMissions", args={
+            "frame": frame,
+        })
+        return baml_py.BamlStream[typing.List["stream_types.MissionSuggestion"], typing.List["types.MissionSuggestion"]](
+          __result__,
+          lambda x: typing.cast(typing.List["stream_types.MissionSuggestion"], x.cast_to(types, types, stream_types, True, __runtime__)),
+          lambda x: typing.cast(typing.List["types.MissionSuggestion"], x.cast_to(types, types, stream_types, False, __runtime__)),
+          __ctx__,
+        )
     
 
 class BamlHttpRequestClient:
@@ -167,6 +194,13 @@ class BamlHttpRequestClient:
             "mission_text": mission_text,
         }, mode="request")
         return __result__
+    async def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.baml_py.HTTPRequest:
+        __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SuggestMissions", args={
+            "frame": frame,
+        }, mode="request")
+        return __result__
     
 
 class BamlHttpStreamRequestClient:
@@ -189,6 +223,13 @@ class BamlHttpStreamRequestClient:
             "mission_text": mission_text,
         }, mode="stream")
         return __result__
+    async def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.baml_py.HTTPRequest:
+        __result__ = await self.__options.merge_options(baml_options).create_http_request_async(function_name="SuggestMissions", args={
+            "frame": frame,
+        }, mode="stream")
+        return __result__
     
 
 b = BamlAsyncClient(DoNotUseDirectlyCallManager({}))

baml_client/inlinedbaml.py

@@ -14,7 +14,7 @@ _file_map = {
 
     "clients.baml": "// ISR LLM clients\n\nclient<llm> GPT4oMini {\n  provider openai\n  retry_policy Retry\n  options {\n    model \"gpt-4o-mini\"\n    api_key env.OPENAI_API_KEY\n    temperature 0.1\n  }\n}\n\nclient<llm> GPT4o {\n  provider openai\n  retry_policy Retry\n  options {\n    model \"gpt-4o\"\n    api_key env.OPENAI_API_KEY\n    temperature 0.2\n  }\n}\n\nretry_policy Retry {\n  max_retries 2\n  strategy {\n    type exponential_backoff\n    delay_ms 500\n    multiplier 2.0\n    max_delay_ms 5000\n  }\n}\n",
     "generators.baml": "// This helps use auto generate libraries you can use in the language of\n// your choice. You can have multiple generators if you use multiple languages.\n// Just ensure that the output_dir is different for each generator.\ngenerator target {\n    // Valid values: \"python/pydantic\", \"typescript\", \"go\", \"rust\", \"ruby/sorbet\", \"rest/openapi\"\n    output_type \"python/pydantic\"\n\n    // Where the generated code will be saved (relative to baml_src/)\n    output_dir \"../\"\n\n    // The version of the BAML package you have installed (e.g. same version as your baml-py or @boundaryml/baml).\n    // The BAML VSCode extension version should also match this version.\n    version \"0.220.0\"\n\n    // Valid values: \"sync\", \"async\"\n    // This controls what `b.FunctionName()` will be (sync or async).\n    default_client_mode sync\n}\n",
-    "isr.baml": "// ISR Mission Planning & Assessment Functions\n\n// ── Mission Planning ─────────────────────────────────────────────\n// Takes a free-form mission objective and produces:\n// 1. Concrete object class queries for the detector (YOLO/DETR/GDINO)\n// 2. A refined mission statement for downstream assessment\n\nclass MissionPlan {\n  detector_queries string[] @description(\"ONLY the object classes directly targeted by the mission. Use COCO class names (person, car, truck, bicycle, motorcycle, bus, dog, cat, etc.). Include ONLY classes the mission explicitly asks to find — do NOT pad with tangentially related classes. Typically 1-4 items.\")\n  refined_mission string @description(\"A clear, one-sentence restatement of the mission objective that a downstream analyst LLM will evaluate each detection against.\")\n  reasoning string @description(\"Brief explanation of why these queries were chosen.\")\n}\n\nfunction PlanMission(mission_text: string) -> MissionPlan {\n  client GPT4oMini\n  prompt #\"\n    You are an ISR (Intelligence, Surveillance, Reconnaissance) mission planner.\n\n    Your job is to decide what BROAD object categories a visual detector (YOLO / DETR / Grounding DINO) should look for.\n\n    IMPORTANT — separation of concerns:\n    - The DETECTOR can only recognize broad visual categories (person, car, truck, bus, etc.).\n      It CANNOT judge intent, context, or mission-specific conditions.\n    - A DOWNSTREAM ANALYST LLM will later examine each detection and decide whether it\n      satisfies the mission. That is where nuanced judgment happens.\n\n    Therefore:\n    - Output ONLY the object classes the mission EXPLICITLY targets. Keep the list MINIMAL.\n    - NEVER pad with \"context\" classes that happen to appear in the scene but are not the\n      mission target. Every extra class wastes GPU cycles on irrelevant detections.\n    - NEVER output mission-specific or subjective labels (e.g. \"stranded person\", \"cargo truck\",\n      \"suspicious vehicle\"). The detector cannot distinguish these from their parent category.\n    - Use standard COCO class names when possible: person, car, truck, bus, motorcycle, bicycle,\n      dog, cat, horse, sheep, cow, elephant, bear, zebra, giraffe, bird,\n      boat, airplane, backpack, suitcase, handbag, umbrella, etc.\n\n    KEY RULE — ask \"Is the mission trying to FIND this class of object?\"\n    - YES → include it.  NO → leave it out, even if it appears in the scene.\n\n    Examples:\n    - \"identify person stranded on rooftop\" → detector_queries: [\"person\"]\n      (mission targets people, NOT rooftops. Downstream LLM judges context.)\n    - \"find motorcycles lane-splitting between traffic\" → detector_queries: [\"motorcycle\"]\n      (mission targets motorcycles, NOT the surrounding cars/trucks.)\n    - \"identify vehicles that can carry heavy cargos\" → detector_queries: [\"truck\", \"bus\", \"car\"]\n      (mission targets vehicles broadly — multiple vehicle classes needed.)\n    - \"find abandoned luggage in airport\" → detector_queries: [\"suitcase\", \"backpack\", \"handbag\"]\n      (mission targets luggage — multiple luggage classes needed.)\n\n    Mission objective: \"{{ mission_text }}\"\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n\n// ── Detection Assessment ─────────────────────────────────────────\n// Replaces hand-rolled JSON parsing with type-safe BAML output\n\nclass DetectionInfo {\n  track_id string\n  class_label string\n  confidence float @description(\"Detection model confidence 0.0-1.0. Lower values mean the detector is less sure about the class.\")\n  center_x float @description(\"Horizontal position in frame, 0.0=left edge, 1.0=right edge\")\n  center_y float @description(\"Vertical position in frame, 0.0=top edge, 1.0=bottom edge\")\n  speed_kph float\n  direction string @description(\"Clock direction (e.g. '3 o'clock' for rightward) or 'stationary'\")\n}\n\nclass DetectionVerdict {\n  track_id string\n  mission_relevant bool @description(\"Does this broad object CLASS relate to the mission? e.g. 'person' is relevant to a rescue mission, 'dog' is not.\")\n  satisfies bool? @description(\"Based on visual context, does THIS SPECIFIC detection actually meet the mission criteria? e.g. a person on a rooftop satisfies 'find stranded people', but a person walking on a street does not. null if the image is too ambiguous to judge.\")\n  reason string @description(\"1-2 sentences: what you observe in the image that supports your verdict. Reference specific visual cues (location, posture, surroundings, motion).\")\n  features map<string, string> @description(\"2-5 observable properties from the image relevant to the mission. e.g. location: 'rooftop', posture: 'standing', surroundings: 'floodwater', vehicle_type: 'flatbed truck'\")\n}\n\n// ── PlanMission Tests ────────────────────────────────────────────\n\ntest HeavyCargoVehicles {\n  functions [PlanMission]\n  args {\n    mission_text \"identify vehicles that can carry heavy cargos\"\n  }\n  // Mission targets vehicles broadly — multiple vehicle types needed\n  @@assert( {{ this.detector_queries|length >= 2 }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ \"cargo\" not in this.detector_queries|join(\" \") }} )\n  // No non-vehicle padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest PersonOnRooftop {\n  functions [PlanMission]\n  args {\n    mission_text \"identify person stranded on rooftop\"\n  }\n  // Mission targets person only — rooftop is context for assessor\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"stranded\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"rooftop\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest AbandonedLuggage {\n  functions [PlanMission]\n  args {\n    mission_text \"find abandoned luggage in airport terminal\"\n  }\n  // Mission targets luggage — multiple luggage classes needed\n  @@assert( {{ this.detector_queries|length >= 2 }} )\n  @@assert( {{ this.detector_queries|length <= 4 }} )\n  @@assert( {{ \"abandoned\" not in this.detector_queries|join(\" \") }} )\n  // No non-luggage padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"airplane\" not in this.detector_queries }} )\n}\n\ntest VehiclesBlockingHighway {\n  functions [PlanMission]\n  args {\n    mission_text \"locate vehicles blocking the highway exit ramp\"\n  }\n  // Mission targets vehicles — multiple vehicle types needed\n  @@assert( {{ \"car\" in this.detector_queries }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"blocking\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"highway\" not in this.detector_queries|join(\" \") }} )\n  // No non-vehicle padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest PeopleJaywalking {\n  functions [PlanMission]\n  args {\n    mission_text \"identify people jaywalking across the main road\"\n  }\n  // Mission targets people only — road/vehicles are context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"jaywalking\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"road\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"bus\" not in this.detector_queries }} )\n  @@assert( {{ \"bicycle\" not in this.detector_queries }} )\n}\n\ntest BoatsNearRestrictedDock {\n  functions [PlanMission]\n  args {\n    mission_text \"find boats anchored near the restricted dock area\"\n  }\n  // Mission targets boats only\n  @@assert( {{ \"boat\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"anchored\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"restricted\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"dock\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest DogsOffLeash {\n  functions [PlanMission]\n  args {\n    mission_text \"detect dogs off-leash in the public park\"\n  }\n  // Mission targets dogs only — leash/park are context\n  @@assert( {{ \"dog\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"leash\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"off-leash\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"park\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"cat\" not in this.detector_queries }} )\n}\n\ntest DeliveryTrucksInDriveways {\n  functions [PlanMission]\n  args {\n    mission_text \"locate delivery trucks parked in residential driveways\"\n  }\n  // Mission targets trucks only — delivery/residential are context\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"delivery\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"residential\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"driveway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest CyclistsOnSidewalk {\n  functions [PlanMission]\n  args {\n    mission_text \"identify cyclists riding on the pedestrian sidewalk\"\n  }\n  // Mission targets cyclists = bicycle + person\n  @@assert( {{ \"bicycle\" in this.detector_queries }} )\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 3 }} )\n  @@assert( {{ \"cyclist\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"sidewalk\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"pedestrian\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n}\n\ntest PersonsLoiteringNearExit {\n  functions [PlanMission]\n  args {\n    mission_text \"find persons loitering near the emergency exit\"\n  }\n  // Mission targets persons only — exit is context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"loitering\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"emergency\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"door\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest OverturnedVehicles {\n  functions [PlanMission]\n  args {\n    mission_text \"detect overturned vehicles on the freeway\"\n  }\n  // Mission targets vehicles — multiple vehicle types needed\n  @@assert( {{ \"car\" in this.detector_queries }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"overturned\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"freeway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest AircraftOutsideApron {\n  functions [PlanMission]\n  args {\n    mission_text \"locate aircraft parked outside designated apron area\"\n  }\n  // Mission targets aircraft only\n  @@assert( {{ \"airplane\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"designated\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"apron\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest PersonsWithBackpacksNearFence {\n  functions [PlanMission]\n  args {\n    mission_text \"identify persons carrying large backpacks near the perimeter fence\"\n  }\n  // Mission targets persons + backpacks\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ \"backpack\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 3 }} )\n  @@assert( {{ \"perimeter\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"fence\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"carrying\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest MotorcyclesLaneSplitting {\n  functions [PlanMission]\n  args {\n    mission_text \"find motorcycles lane-splitting between traffic\"\n  }\n  // Mission targets motorcycles only — surrounding traffic is context\n  @@assert( {{ \"motorcycle\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"lane-splitting\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"traffic\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest CattleOnRoadway {\n  functions [PlanMission]\n  args {\n    mission_text \"detect cattle that have crossed onto the roadway\"\n  }\n  // Mission targets cattle (COCO: cow) only\n  @@assert( {{ \"cow\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"cattle\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"roadway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"crossed\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest BusesAtUnauthorizedStops {\n  functions [PlanMission]\n  args {\n    mission_text \"locate buses stopped at unauthorized pickup points\"\n  }\n  // Mission targets buses only\n  @@assert( {{ \"bus\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"unauthorized\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"pickup\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest PeopleSwimmingInRestrictedArea {\n  functions [PlanMission]\n  args {\n    mission_text \"identify people swimming in the restricted waterway\"\n  }\n  // Mission targets people only — waterway is context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"swimming\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"restricted\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"waterway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"boat\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest TrucksWithOversizedLoads {\n  functions [PlanMission]\n  args {\n    mission_text \"find trucks carrying visible oversized loads on the bridge\"\n  }\n  // Mission targets trucks only — bridge/load are context\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"oversized\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"cargo\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"bridge\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\n\n// ── Detection Assessment ─────────────────────────────────────────\n// Replaces hand-rolled JSON parsing with type-safe BAML output\n\nfunction AssessDetections(mission: string, detections: DetectionInfo[], frame_image: image) -> DetectionVerdict[] {\n  client GPT4oMini\n  prompt #\"\n    {{ _.role(\"system\") }}\n    You are an ISR analyst. A broad-net detector found candidate objects. Your job: decide\n    which ones actually satisfy the mission using the image and metadata below.\n\n    Reading the metadata:\n    - **confidence**: detector certainty (0-1). Below 0.4 = likely false positive — verify visually before trusting the class label.\n    - **position**: (x, y) in frame where (0,0)=top-left, (1,1)=bottom-right. Use this to locate each detection in the image.\n    - **speed/direction**: from multi-frame tracking. speed=0 means stationary. Direction uses clock notation (12=up, 3=right, 6=down, 9=left).\n\n    For each detection, decide:\n    - **mission_relevant**: Does this object CLASS relate to the mission? (person vs dog, not this specific instance.)\n    - **satisfies**: Based on what you SEE in the image — does this specific detection meet the mission criteria? Consider location, posture, surroundings, object subtype. Set null only when genuinely ambiguous.\n    - **reason**: Cite what you observe in the image. No generic statements.\n    - **features**: 2-5 observable properties from the image relevant to the mission.\n\n    {{ _.role(\"user\") }}\n    Mission: \"{{ mission }}\"\n\n    Detected objects:\n    {% for d in detections %}\n    - {{ d.track_id }}: {{ d.class_label }} (conf={{ d.confidence }}) at ({{ d.center_x }}, {{ d.center_y }}), {{ d.speed_kph }}kph {{ d.direction }}\n    {% endfor %}\n\n    {{ frame_image }}\n\n    Assess every detection above.\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n// ── AssessDetections Tests ───────────────────────────────────────\n// SAR flood scene: person + dog on rooftop, surrounded by floodwater\n\n// Wide net: detector found person + dog. Assessor must judge context from image.\ntest SAR_PersonOnRooftop {\n  functions [AssessDetections]\n  args {\n    mission \"identify person stranded on rooftop needing rescue\"\n    detections [\n      {\n        track_id \"T01\"\n        class_label \"person\"\n        confidence 0.87\n        center_x 0.45\n        center_y 0.35\n        speed_kph 0.0\n        direction \"stationary\"\n      },\n      {\n        track_id \"T02\"\n        class_label \"dog\"\n        confidence 0.72\n        center_x 0.52\n        center_y 0.40\n        speed_kph 0.0\n        direction \"stationary\"\n      }\n    ]\n    frame_image {\n      file \"fixtures/sar_rooftop.jpg\"\n      media_type \"image/jpeg\"\n    }\n  }\n  @@assert( {{ this|length == 2 }} )\n  // Person on rooftop: class is relevant AND visual context satisfies mission\n  @@assert( {{ this[0].track_id == \"T01\" }} )\n  @@assert( {{ this[0].mission_relevant == true }} )\n  @@assert( {{ this[0].satisfies == true }} )\n  // Dog: class is not relevant to \"person stranded\" mission\n  @@assert( {{ this[1].track_id == \"T02\" }} )\n  @@assert( {{ this[1].mission_relevant == false }} )\n}\n\n// Wide net caught person + dog, but mission is about vehicles — both should be irrelevant.\ntest NeitherPersonNorDogIsVehicle {\n  functions [AssessDetections]\n  args {\n    mission \"identify vehicles capable of transporting heavy cargo\"\n    detections [\n      {\n        track_id \"T01\"\n        class_label \"person\"\n        confidence 0.87\n        center_x 0.45\n        center_y 0.35\n        speed_kph 0.0\n        direction \"stationary\"\n      },\n      {\n        track_id \"T02\"\n        class_label \"dog\"\n        confidence 0.72\n        center_x 0.52\n        center_y 0.40\n        speed_kph 0.0\n        direction \"stationary\"\n      }\n    ]\n    frame_image {\n      file \"fixtures/sar_rooftop.jpg\"\n      media_type \"image/jpeg\"\n    }\n  }\n  @@assert( {{ this|length == 2 }} )\n  // Neither person nor dog is a vehicle — both mission_relevant=false\n  @@assert( {{ this[0].mission_relevant == false }} )\n  @@assert( {{ this[1].mission_relevant == false }} )\n  @@assert( {{ this[0].satisfies != true }} )\n  @@assert( {{ this[1].satisfies != true }} )\n}\n",
+    "isr.baml": "// ISR Mission Planning & Assessment Functions\n\n// ── Mission Planning ─────────────────────────────────────────────\n// Takes a free-form mission objective and produces:\n// 1. Concrete object class queries for the detector (YOLO/DETR/GDINO)\n// 2. A refined mission statement for downstream assessment\n\nclass MissionPlan {\n  detector_queries string[] @description(\"ONLY the object classes directly targeted by the mission. Use COCO class names (person, car, truck, bicycle, motorcycle, bus, dog, cat, etc.). Include ONLY classes the mission explicitly asks to find — do NOT pad with tangentially related classes. Typically 1-4 items.\")\n  refined_mission string @description(\"A clear, one-sentence restatement of the mission objective that a downstream analyst LLM will evaluate each detection against.\")\n  reasoning string @description(\"Brief explanation of why these queries were chosen.\")\n}\n\nfunction PlanMission(mission_text: string) -> MissionPlan {\n  client GPT4oMini\n  prompt #\"\n    You are an ISR (Intelligence, Surveillance, Reconnaissance) mission planner.\n\n    Your job is to decide what BROAD object categories a visual detector (YOLO / DETR / Grounding DINO) should look for.\n\n    IMPORTANT — separation of concerns:\n    - The DETECTOR can only recognize broad visual categories (person, car, truck, bus, etc.).\n      It CANNOT judge intent, context, or mission-specific conditions.\n    - A DOWNSTREAM ANALYST LLM will later examine each detection and decide whether it\n      satisfies the mission. That is where nuanced judgment happens.\n\n    Therefore:\n    - Output ONLY the object classes the mission EXPLICITLY targets. Keep the list MINIMAL.\n    - NEVER pad with \"context\" classes that happen to appear in the scene but are not the\n      mission target. Every extra class wastes GPU cycles on irrelevant detections.\n    - NEVER output mission-specific or subjective labels (e.g. \"stranded person\", \"cargo truck\",\n      \"suspicious vehicle\"). The detector cannot distinguish these from their parent category.\n    - Use standard COCO class names when possible: person, car, truck, bus, motorcycle, bicycle,\n      dog, cat, horse, sheep, cow, elephant, bear, zebra, giraffe, bird,\n      boat, airplane, backpack, suitcase, handbag, umbrella, etc.\n\n    KEY RULE — ask \"Is the mission trying to FIND this class of object?\"\n    - YES → include it.  NO → leave it out, even if it appears in the scene.\n\n    Examples:\n    - \"identify person stranded on rooftop\" → detector_queries: [\"person\"]\n      (mission targets people, NOT rooftops. Downstream LLM judges context.)\n    - \"find motorcycles lane-splitting between traffic\" → detector_queries: [\"motorcycle\"]\n      (mission targets motorcycles, NOT the surrounding cars/trucks.)\n    - \"identify vehicles that can carry heavy cargos\" → detector_queries: [\"truck\", \"bus\", \"car\"]\n      (mission targets vehicles broadly — multiple vehicle classes needed.)\n    - \"find abandoned luggage in airport\" → detector_queries: [\"suitcase\", \"backpack\", \"handbag\"]\n      (mission targets luggage — multiple luggage classes needed.)\n\n    Mission objective: \"{{ mission_text }}\"\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n\n// ── Mission Suggestion ──────────────────────────────────────────\n// Analyzes a video frame and suggests actionable ISR missions\n\nclass MissionSuggestion {\n  mission string @description(\"A concise, natural-language mission objective. Written as an imperative command, e.g. 'Track pedestrians crossing the intersection'. 8-15 words.\")\n  reasoning string @description(\"One sentence: what you see in the image that makes this mission feasible and interesting.\")\n}\n\nfunction SuggestMissions(frame: image) -> MissionSuggestion[] {\n  client GPT4oMini\n  prompt #\"\n    {{ _.role(\"system\") }}\n    You are an ISR (Intelligence, Surveillance, Reconnaissance) mission advisor.\n    Given a single video frame, suggest 3-5 interesting surveillance missions that\n    a detector + analyst pipeline could realistically accomplish.\n\n    RULES — what makes a good suggestion:\n    1. **Visually grounded**: Every suggestion must reference objects you can ACTUALLY SEE\n       in the frame. Do not hallucinate objects that are not there.\n    2. **Detector-feasible**: The mission must target objects a COCO-trained detector can\n       find (person, car, truck, bus, motorcycle, bicycle, boat, dog, cat, airplane, etc.).\n       Do not suggest missions requiring detection of abstract concepts, emotions, or\n       objects not in standard detection models.\n    3. **Analyst-decidable**: The mission criteria must be something a downstream vision LLM\n       can judge from a single annotated frame — spatial position, object state, proximity,\n       relative size, posture, motion direction. Avoid criteria needing temporal history,\n       audio, or information not visible in images.\n    4. **Simple and clear**: Each mission should be a single, unambiguous objective.\n       Not compound (\"find X and also Y\"). 8-15 words.\n    5. **Interesting variety**: Vary the missions — mix counting, tracking, spatial\n       monitoring, and anomaly detection. Don't repeat the same pattern.\n\n    AVOID these — they sound good but fail in practice:\n    - Missions about intent or mental state (\"find suspicious people\", \"detect distressed individuals\")\n    - Missions requiring world knowledge not in the image (\"unauthorized vehicles\", \"restricted zones\")\n    - Missions about very small or occluded objects the detector would miss\n    - Missions that are trivially satisfied by everything in the scene (\"detect all objects\")\n\n    GOOD examples for different scene types:\n    - Street: \"Track vehicles stopped at the intersection\" / \"Monitor pedestrians jaywalking across the road\"\n    - Parking lot: \"Identify cars parked in the fire lane\" / \"Count trucks in the loading area\"\n    - Aerial: \"Locate boats near the shoreline\" / \"Track vehicles on the highway bridge\"\n    - Indoor: \"Monitor people near the emergency exits\" / \"Track shopping carts left in the walkway\"\n\n    {{ _.role(\"user\") }}\n    Analyze this video frame and suggest 3-5 feasible ISR missions:\n\n    {{ frame }}\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n\n// ── Detection Assessment ─────────────────────────────────────────\n// Replaces hand-rolled JSON parsing with type-safe BAML output\n\nclass DetectionInfo {\n  track_id string\n  class_label string\n  confidence float @description(\"Detection model confidence 0.0-1.0. Lower values mean the detector is less sure about the class.\")\n  center_x float @description(\"Horizontal position in frame, 0.0=left edge, 1.0=right edge\")\n  center_y float @description(\"Vertical position in frame, 0.0=top edge, 1.0=bottom edge\")\n  speed_kph float\n  direction string @description(\"Clock direction (e.g. '3 o'clock' for rightward) or 'stationary'\")\n}\n\nclass DetectionVerdict {\n  track_id string\n  mission_relevant bool @description(\"Does this broad object CLASS relate to the mission? e.g. 'person' is relevant to a rescue mission, 'dog' is not.\")\n  satisfies bool? @description(\"Based on visual context, does THIS SPECIFIC detection actually meet the mission criteria? e.g. a person on a rooftop satisfies 'find stranded people', but a person walking on a street does not. null if the image is too ambiguous to judge.\")\n  reason string @description(\"1-2 sentences: what you observe in the image that supports your verdict. Reference specific visual cues (location, posture, surroundings, motion).\")\n  features map<string, string> @description(\"2-5 observable properties from the image relevant to the mission. e.g. location: 'rooftop', posture: 'standing', surroundings: 'floodwater', vehicle_type: 'flatbed truck'\")\n}\n\n// ── PlanMission Tests ────────────────────────────────────────────\n\ntest HeavyCargoVehicles {\n  functions [PlanMission]\n  args {\n    mission_text \"identify vehicles that can carry heavy cargos\"\n  }\n  // Mission targets vehicles broadly — multiple vehicle types needed\n  @@assert( {{ this.detector_queries|length >= 2 }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ \"cargo\" not in this.detector_queries|join(\" \") }} )\n  // No non-vehicle padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest PersonOnRooftop {\n  functions [PlanMission]\n  args {\n    mission_text \"identify person stranded on rooftop\"\n  }\n  // Mission targets person only — rooftop is context for assessor\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"stranded\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"rooftop\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest AbandonedLuggage {\n  functions [PlanMission]\n  args {\n    mission_text \"find abandoned luggage in airport terminal\"\n  }\n  // Mission targets luggage — multiple luggage classes needed\n  @@assert( {{ this.detector_queries|length >= 2 }} )\n  @@assert( {{ this.detector_queries|length <= 4 }} )\n  @@assert( {{ \"abandoned\" not in this.detector_queries|join(\" \") }} )\n  // No non-luggage padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"airplane\" not in this.detector_queries }} )\n}\n\ntest VehiclesBlockingHighway {\n  functions [PlanMission]\n  args {\n    mission_text \"locate vehicles blocking the highway exit ramp\"\n  }\n  // Mission targets vehicles — multiple vehicle types needed\n  @@assert( {{ \"car\" in this.detector_queries }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"blocking\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"highway\" not in this.detector_queries|join(\" \") }} )\n  // No non-vehicle padding\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest PeopleJaywalking {\n  functions [PlanMission]\n  args {\n    mission_text \"identify people jaywalking across the main road\"\n  }\n  // Mission targets people only — road/vehicles are context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"jaywalking\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"road\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"bus\" not in this.detector_queries }} )\n  @@assert( {{ \"bicycle\" not in this.detector_queries }} )\n}\n\ntest BoatsNearRestrictedDock {\n  functions [PlanMission]\n  args {\n    mission_text \"find boats anchored near the restricted dock area\"\n  }\n  // Mission targets boats only\n  @@assert( {{ \"boat\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"anchored\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"restricted\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"dock\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest DogsOffLeash {\n  functions [PlanMission]\n  args {\n    mission_text \"detect dogs off-leash in the public park\"\n  }\n  // Mission targets dogs only — leash/park are context\n  @@assert( {{ \"dog\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"leash\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"off-leash\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"park\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"cat\" not in this.detector_queries }} )\n}\n\ntest DeliveryTrucksInDriveways {\n  functions [PlanMission]\n  args {\n    mission_text \"locate delivery trucks parked in residential driveways\"\n  }\n  // Mission targets trucks only — delivery/residential are context\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"delivery\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"residential\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"driveway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest CyclistsOnSidewalk {\n  functions [PlanMission]\n  args {\n    mission_text \"identify cyclists riding on the pedestrian sidewalk\"\n  }\n  // Mission targets cyclists = bicycle + person\n  @@assert( {{ \"bicycle\" in this.detector_queries }} )\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 3 }} )\n  @@assert( {{ \"cyclist\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"sidewalk\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"pedestrian\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n}\n\ntest PersonsLoiteringNearExit {\n  functions [PlanMission]\n  args {\n    mission_text \"find persons loitering near the emergency exit\"\n  }\n  // Mission targets persons only — exit is context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"loitering\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"emergency\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"door\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest OverturnedVehicles {\n  functions [PlanMission]\n  args {\n    mission_text \"detect overturned vehicles on the freeway\"\n  }\n  // Mission targets vehicles — multiple vehicle types needed\n  @@assert( {{ \"car\" in this.detector_queries }} )\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 5 }} )\n  @@assert( {{ \"overturned\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"freeway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"dog\" not in this.detector_queries }} )\n}\n\ntest AircraftOutsideApron {\n  functions [PlanMission]\n  args {\n    mission_text \"locate aircraft parked outside designated apron area\"\n  }\n  // Mission targets aircraft only\n  @@assert( {{ \"airplane\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"designated\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"apron\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest PersonsWithBackpacksNearFence {\n  functions [PlanMission]\n  args {\n    mission_text \"identify persons carrying large backpacks near the perimeter fence\"\n  }\n  // Mission targets persons + backpacks\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ \"backpack\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 3 }} )\n  @@assert( {{ \"perimeter\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"fence\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"carrying\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest MotorcyclesLaneSplitting {\n  functions [PlanMission]\n  args {\n    mission_text \"find motorcycles lane-splitting between traffic\"\n  }\n  // Mission targets motorcycles only — surrounding traffic is context\n  @@assert( {{ \"motorcycle\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"lane-splitting\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"traffic\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest CattleOnRoadway {\n  functions [PlanMission]\n  args {\n    mission_text \"detect cattle that have crossed onto the roadway\"\n  }\n  // Mission targets cattle (COCO: cow) only\n  @@assert( {{ \"cow\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"cattle\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"roadway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"crossed\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest BusesAtUnauthorizedStops {\n  functions [PlanMission]\n  args {\n    mission_text \"locate buses stopped at unauthorized pickup points\"\n  }\n  // Mission targets buses only\n  @@assert( {{ \"bus\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"unauthorized\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"pickup\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"truck\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\ntest PeopleSwimmingInRestrictedArea {\n  functions [PlanMission]\n  args {\n    mission_text \"identify people swimming in the restricted waterway\"\n  }\n  // Mission targets people only — waterway is context\n  @@assert( {{ \"person\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"swimming\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"restricted\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"waterway\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"boat\" not in this.detector_queries }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n}\n\ntest TrucksWithOversizedLoads {\n  functions [PlanMission]\n  args {\n    mission_text \"find trucks carrying visible oversized loads on the bridge\"\n  }\n  // Mission targets trucks only — bridge/load are context\n  @@assert( {{ \"truck\" in this.detector_queries }} )\n  @@assert( {{ this.detector_queries|length <= 2 }} )\n  @@assert( {{ \"oversized\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"cargo\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"bridge\" not in this.detector_queries|join(\" \") }} )\n  @@assert( {{ \"car\" not in this.detector_queries }} )\n  @@assert( {{ \"person\" not in this.detector_queries }} )\n}\n\n\n// ── Detection Assessment ─────────────────────────────────────────\n// Replaces hand-rolled JSON parsing with type-safe BAML output\n\nfunction AssessDetections(mission: string, detections: DetectionInfo[], frame_image: image) -> DetectionVerdict[] {\n  client GPT4oMini\n  prompt #\"\n    {{ _.role(\"system\") }}\n    You are an ISR analyst. A broad-net detector found candidate objects. Your job: decide\n    which ones actually satisfy the mission using the image and metadata below.\n\n    Reading the metadata:\n    - **confidence**: detector certainty (0-1). Below 0.4 = likely false positive — verify visually before trusting the class label.\n    - **position**: (x, y) in frame where (0,0)=top-left, (1,1)=bottom-right. Use this to locate each detection in the image.\n    - **speed/direction**: from multi-frame tracking. speed=0 means stationary. Direction uses clock notation (12=up, 3=right, 6=down, 9=left).\n\n    For each detection, decide:\n    - **mission_relevant**: Does this object CLASS relate to the mission? (person vs dog, not this specific instance.)\n    - **satisfies**: Based on what you SEE in the image — does this specific detection meet the mission criteria? Consider location, posture, surroundings, object subtype. Set null only when genuinely ambiguous.\n    - **reason**: Cite what you observe in the image. No generic statements.\n    - **features**: 2-5 observable properties from the image relevant to the mission.\n\n    {{ _.role(\"user\") }}\n    Mission: \"{{ mission }}\"\n\n    Detected objects:\n    {% for d in detections %}\n    - {{ d.track_id }}: {{ d.class_label }} (conf={{ d.confidence }}) at ({{ d.center_x }}, {{ d.center_y }}), {{ d.speed_kph }}kph {{ d.direction }}\n    {% endfor %}\n\n    {{ frame_image }}\n\n    Assess every detection above.\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n// ── AssessDetections Tests ───────────────────────────────────────\n// SAR flood scene: person + dog on rooftop, surrounded by floodwater\n\n// Wide net: detector found person + dog. Assessor must judge context from image.\ntest SAR_PersonOnRooftop {\n  functions [AssessDetections]\n  args {\n    mission \"identify person stranded on rooftop needing rescue\"\n    detections [\n      {\n        track_id \"T01\"\n        class_label \"person\"\n        confidence 0.87\n        center_x 0.45\n        center_y 0.35\n        speed_kph 0.0\n        direction \"stationary\"\n      },\n      {\n        track_id \"T02\"\n        class_label \"dog\"\n        confidence 0.72\n        center_x 0.52\n        center_y 0.40\n        speed_kph 0.0\n        direction \"stationary\"\n      }\n    ]\n    frame_image {\n      file \"fixtures/sar_rooftop.jpg\"\n      media_type \"image/jpeg\"\n    }\n  }\n  @@assert( {{ this|length == 2 }} )\n  // Person on rooftop: class is relevant AND visual context satisfies mission\n  @@assert( {{ this[0].track_id == \"T01\" }} )\n  @@assert( {{ this[0].mission_relevant == true }} )\n  @@assert( {{ this[0].satisfies == true }} )\n  // Dog: class is not relevant to \"person stranded\" mission\n  @@assert( {{ this[1].track_id == \"T02\" }} )\n  @@assert( {{ this[1].mission_relevant == false }} )\n}\n\n// Wide net caught person + dog, but mission is about vehicles — both should be irrelevant.\ntest NeitherPersonNorDogIsVehicle {\n  functions [AssessDetections]\n  args {\n    mission \"identify vehicles capable of transporting heavy cargo\"\n    detections [\n      {\n        track_id \"T01\"\n        class_label \"person\"\n        confidence 0.87\n        center_x 0.45\n        center_y 0.35\n        speed_kph 0.0\n        direction \"stationary\"\n      },\n      {\n        track_id \"T02\"\n        class_label \"dog\"\n        confidence 0.72\n        center_x 0.52\n        center_y 0.40\n        speed_kph 0.0\n        direction \"stationary\"\n      }\n    ]\n    frame_image {\n      file \"fixtures/sar_rooftop.jpg\"\n      media_type \"image/jpeg\"\n    }\n  }\n  @@assert( {{ this|length == 2 }} )\n  // Neither person nor dog is a vehicle — both mission_relevant=false\n  @@assert( {{ this[0].mission_relevant == false }} )\n  @@assert( {{ this[1].mission_relevant == false }} )\n  @@assert( {{ this[0].satisfies != true }} )\n  @@assert( {{ this[1].satisfies != true }} )\n}\n",
 }
 
 def get_baml_files():

baml_client/parser.py

@@ -35,6 +35,12 @@ class LlmResponseParser:
         __result__ = self.__options.merge_options(baml_options).parse_response(function_name="PlanMission", llm_response=llm_response, mode="request")
         return typing.cast(types.MissionPlan, __result__)
 
+    def SuggestMissions(
+        self, llm_response: str, baml_options: BamlCallOptions = {},
+    ) -> typing.List["types.MissionSuggestion"]:
+        __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SuggestMissions", llm_response=llm_response, mode="request")
+        return typing.cast(typing.List["types.MissionSuggestion"], __result__)
+
     
 
 class LlmStreamParser:
@@ -55,4 +61,10 @@ class LlmStreamParser:
         __result__ = self.__options.merge_options(baml_options).parse_response(function_name="PlanMission", llm_response=llm_response, mode="stream")
         return typing.cast(stream_types.MissionPlan, __result__)
 
+    def SuggestMissions(
+        self, llm_response: str, baml_options: BamlCallOptions = {},
+    ) -> typing.List["stream_types.MissionSuggestion"]:
+        __result__ = self.__options.merge_options(baml_options).parse_response(function_name="SuggestMissions", llm_response=llm_response, mode="stream")
+        return typing.cast(typing.List["stream_types.MissionSuggestion"], __result__)
+
     

baml_client/stream_types.py

@@ -23,7 +23,7 @@ class StreamState(BaseModel, typing.Generic[StreamStateValueT]):
     value: StreamStateValueT
     state: typing_extensions.Literal["Pending", "Incomplete", "Complete"]
 # #########################################################################
-# Generated classes (3)
+# Generated classes (4)
 # #########################################################################
 
 class DetectionInfo(BaseModel):
@@ -47,6 +47,10 @@ class MissionPlan(BaseModel):
     refined_mission: typing.Optional[str] = Field(default=None, description='A clear, one-sentence restatement of the mission objective that a downstream analyst LLM will evaluate each detection against.')
     reasoning: typing.Optional[str] = Field(default=None, description='Brief explanation of why these queries were chosen.')
 
+class MissionSuggestion(BaseModel):
+    mission: typing.Optional[str] = Field(default=None, description='A concise, natural-language mission objective. Written as an imperative command, e.g. \'Track pedestrians crossing the intersection\'. 8-15 words.')
+    reasoning: typing.Optional[str] = Field(default=None, description='One sentence: what you see in the image that makes this mission feasible and interesting.')
+
 # #########################################################################
 # Generated type aliases (0)
 # #########################################################################

baml_client/sync_client.py

@@ -122,6 +122,20 @@ class BamlSyncClient:
                 "mission_text": mission_text,
             })
             return typing.cast(types.MissionPlan, __result__.cast_to(types, types, stream_types, False, __runtime__))
+    def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> typing.List["types.MissionSuggestion"]:
+        # Check if on_tick is provided
+        if 'on_tick' in baml_options:
+            __stream__ = self.stream.SuggestMissions(frame=frame,
+                baml_options=baml_options)
+            return __stream__.get_final_response()
+        else:
+            # Original non-streaming code
+            __result__ = self.__options.merge_options(baml_options).call_function_sync(function_name="SuggestMissions", args={
+                "frame": frame,
+            })
+            return typing.cast(typing.List["types.MissionSuggestion"], __result__.cast_to(types, types, stream_types, False, __runtime__))
     
 
 
@@ -155,6 +169,18 @@ class BamlStreamClient:
           lambda x: typing.cast(types.MissionPlan, x.cast_to(types, types, stream_types, False, __runtime__)),
           __ctx__,
         )
+    def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.BamlSyncStream[typing.List["stream_types.MissionSuggestion"], typing.List["types.MissionSuggestion"]]:
+        __ctx__, __result__ = self.__options.merge_options(baml_options).create_sync_stream(function_name="SuggestMissions", args={
+            "frame": frame,
+        })
+        return baml_py.BamlSyncStream[typing.List["stream_types.MissionSuggestion"], typing.List["types.MissionSuggestion"]](
+          __result__,
+          lambda x: typing.cast(typing.List["stream_types.MissionSuggestion"], x.cast_to(types, types, stream_types, True, __runtime__)),
+          lambda x: typing.cast(typing.List["types.MissionSuggestion"], x.cast_to(types, types, stream_types, False, __runtime__)),
+          __ctx__,
+        )
     
 
 class BamlHttpRequestClient:
@@ -177,6 +203,13 @@ class BamlHttpRequestClient:
             "mission_text": mission_text,
         }, mode="request")
         return __result__
+    def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.baml_py.HTTPRequest:
+        __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SuggestMissions", args={
+            "frame": frame,
+        }, mode="request")
+        return __result__
     
 
 class BamlHttpStreamRequestClient:
@@ -199,6 +232,13 @@ class BamlHttpStreamRequestClient:
             "mission_text": mission_text,
         }, mode="stream")
         return __result__
+    def SuggestMissions(self, frame: baml_py.Image,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.baml_py.HTTPRequest:
+        __result__ = self.__options.merge_options(baml_options).create_http_request_sync(function_name="SuggestMissions", args={
+            "frame": frame,
+        }, mode="stream")
+        return __result__
     
 
 b = BamlSyncClient(DoNotUseDirectlyCallManager({}))

baml_client/type_builder.py

@@ -20,7 +20,7 @@ from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIM
 class TypeBuilder(type_builder.TypeBuilder):
     def __init__(self):
         super().__init__(classes=set(
-          ["DetectionInfo","DetectionVerdict","MissionPlan",]
+          ["DetectionInfo","DetectionVerdict","MissionPlan","MissionSuggestion",]
         ), enums=set(
           []
         ), runtime=DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME)
@@ -31,7 +31,7 @@ class TypeBuilder(type_builder.TypeBuilder):
 
 
     # #########################################################################
-    # Generated classes 3
+    # Generated classes 4
     # #########################################################################
 
     @property
@@ -46,6 +46,10 @@ class TypeBuilder(type_builder.TypeBuilder):
     def MissionPlan(self) -> "MissionPlanViewer":
         return MissionPlanViewer(self)
 
+    @property
+    def MissionSuggestion(self) -> "MissionSuggestionViewer":
+        return MissionSuggestionViewer(self)
+
 
 
 # #########################################################################
@@ -54,7 +58,7 @@ class TypeBuilder(type_builder.TypeBuilder):
 
 
 # #########################################################################
-# Generated classes 3
+# Generated classes 4
 # #########################################################################
 
 class DetectionInfoAst:
@@ -221,3 +225,46 @@ class MissionPlanProperties:
     
     
 
+
+class MissionSuggestionAst:
+    def __init__(self, tb: type_builder.TypeBuilder):
+        _tb = tb._tb # type: ignore (we know how to use this private attribute)
+        self._bldr = _tb.class_("MissionSuggestion")
+        self._properties: typing.Set[str] = set([  "mission",  "reasoning",  ])
+        self._props = MissionSuggestionProperties(self._bldr, self._properties)
+
+    def type(self) -> baml_py.FieldType:
+        return self._bldr.field()
+
+    @property
+    def props(self) -> "MissionSuggestionProperties":
+        return self._props
+
+
+class MissionSuggestionViewer(MissionSuggestionAst):
+    def __init__(self, tb: type_builder.TypeBuilder):
+        super().__init__(tb)
+
+    
+    def list_properties(self) -> typing.List[typing.Tuple[str, type_builder.ClassPropertyViewer]]:
+        return [(name, type_builder.ClassPropertyViewer(self._bldr.property(name))) for name in self._properties]
+    
+
+
+class MissionSuggestionProperties:
+    def __init__(self, bldr: baml_py.ClassBuilder, properties: typing.Set[str]):
+        self.__bldr = bldr
+        self.__properties = properties # type: ignore (we know how to use this private attribute) # noqa: F821
+
+    
+    
+    @property
+    def mission(self) -> type_builder.ClassPropertyViewer:
+        return type_builder.ClassPropertyViewer(self.__bldr.property("mission"))
+    
+    @property
+    def reasoning(self) -> type_builder.ClassPropertyViewer:
+        return type_builder.ClassPropertyViewer(self.__bldr.property("reasoning"))
+    
+    
+

baml_client/type_map.py

@@ -25,5 +25,8 @@ type_map = {
     "types.MissionPlan": types.MissionPlan,
     "stream_types.MissionPlan": stream_types.MissionPlan,
 
+    "types.MissionSuggestion": types.MissionSuggestion,
+    "stream_types.MissionSuggestion": stream_types.MissionSuggestion,
+
 
 }

baml_client/types.py

@@ -41,7 +41,7 @@ def all_succeeded(checks: typing.Dict[CheckName, Check]) -> bool:
 # #########################################################################
 
 # #########################################################################
-# Generated classes (3)
+# Generated classes (4)
 # #########################################################################
 
 class DetectionInfo(BaseModel):
@@ -65,6 +65,10 @@ class MissionPlan(BaseModel):
     refined_mission: str = Field(description='A clear, one-sentence restatement of the mission objective that a downstream analyst LLM will evaluate each detection against.')
     reasoning: str = Field(description='Brief explanation of why these queries were chosen.')
 
+class MissionSuggestion(BaseModel):
+    mission: str = Field(description='A concise, natural-language mission objective. Written as an imperative command, e.g. \'Track pedestrians crossing the intersection\'. 8-15 words.')
+    reasoning: str = Field(description='One sentence: what you see in the image that makes this mission feasible and interesting.')
+
 # #########################################################################
 # Generated type aliases (0)
 # #########################################################################

baml_src/isr.baml

@@ -54,6 +54,60 @@ function PlanMission(mission_text: string) -> MissionPlan {
 }
 
 
+// ── Mission Suggestion ──────────────────────────────────────────
+// Analyzes a video frame and suggests actionable ISR missions
+
+class MissionSuggestion {
+  mission string @description("A concise, natural-language mission objective. Written as an imperative command, e.g. 'Track pedestrians crossing the intersection'. 8-15 words.")
+  reasoning string @description("One sentence: what you see in the image that makes this mission feasible and interesting.")
+}
+
+function SuggestMissions(frame: image) -> MissionSuggestion[] {
+  client GPT4oMini
+  prompt #"
+    {{ _.role("system") }}
+    You are an ISR (Intelligence, Surveillance, Reconnaissance) mission advisor.
+    Given a single video frame, suggest 3-5 interesting surveillance missions that
+    a detector + analyst pipeline could realistically accomplish.
+
+    RULES — what makes a good suggestion:
+    1. **Visually grounded**: Every suggestion must reference objects you can ACTUALLY SEE
+       in the frame. Do not hallucinate objects that are not there.
+    2. **Detector-feasible**: The mission must target objects a COCO-trained detector can
+       find (person, car, truck, bus, motorcycle, bicycle, boat, dog, cat, airplane, etc.).
+       Do not suggest missions requiring detection of abstract concepts, emotions, or
+       objects not in standard detection models.
+    3. **Analyst-decidable**: The mission criteria must be something a downstream vision LLM
+       can judge from a single annotated frame — spatial position, object state, proximity,
+       relative size, posture, motion direction. Avoid criteria needing temporal history,
+       audio, or information not visible in images.
+    4. **Simple and clear**: Each mission should be a single, unambiguous objective.
+       Not compound ("find X and also Y"). 8-15 words.
+    5. **Interesting variety**: Vary the missions — mix counting, tracking, spatial
+       monitoring, and anomaly detection. Don't repeat the same pattern.
+
+    AVOID these — they sound good but fail in practice:
+    - Missions about intent or mental state ("find suspicious people", "detect distressed individuals")
+    - Missions requiring world knowledge not in the image ("unauthorized vehicles", "restricted zones")
+    - Missions about very small or occluded objects the detector would miss
+    - Missions that are trivially satisfied by everything in the scene ("detect all objects")
+
+    GOOD examples for different scene types:
+    - Street: "Track vehicles stopped at the intersection" / "Monitor pedestrians jaywalking across the road"
+    - Parking lot: "Identify cars parked in the fire lane" / "Count trucks in the loading area"
+    - Aerial: "Locate boats near the shoreline" / "Track vehicles on the highway bridge"
+    - Indoor: "Monitor people near the emergency exits" / "Track shopping carts left in the walkway"
+
+    {{ _.role("user") }}
+    Analyze this video frame and suggest 3-5 feasible ISR missions:
+
+    {{ frame }}
+
+    {{ ctx.output_format }}
+  "#
+}
+
+
 // ── Detection Assessment ─────────────────────────────────────────
 // Replaces hand-rolled JSON parsing with type-safe BAML output
 

demo/index.html

@@ -124,6 +124,7 @@
         <div class="config-group">
           <div class="label">MISSION QUERY</div>
           <input type="text" id="queriesInput" class="config-input" value="person,car,truck" placeholder="e.g., person,car or Monitor unauthorized personnel...">
+          <div id="missionSuggestions" class="suggestion-chips hidden"></div>
         </div>
         <div class="config-row">
           <label class="config-checkbox"><input type="checkbox" id="aiToggle" checked> AI Postprocessing</label>

demo/js/api.js

@@ -8,6 +8,16 @@ const API_BASE = window.location.hostname.includes('hf.space')
 
 // ── Async Data Functions (real backend API) ───────────────────────
 
+async function suggestMissions(frameBlob) {
+  const form = new FormData();
+  form.append('frame', frameBlob, 'frame.jpg');
+  try {
+    const res = await fetch(`${API_BASE}/suggest-missions`, { method: 'POST', body: form });
+    if (!res.ok) return [];
+    return res.json();
+  } catch (err) { console.warn('[ISR] suggestMissions failed:', err); return []; }
+}
+
 async function startDetection(videoFile, config) {
   const form = new FormData();
   form.append('video', videoFile);
@@ -167,6 +177,7 @@ function resolveUrl(path) {
 
 Object.assign(window.ISR, {
   API_BASE,
+  suggestMissions,
   startDetection,
   pollStatus,
   fetchTracks,

demo/js/init.js

@@ -49,6 +49,9 @@ document.addEventListener('DOMContentLoaded', () => {
   // Task 3+4: Start button — real API if video file, mock fallback otherwise
   document.getElementById('startBtn').addEventListener('click', async () => {
     if (STATE.videoFile) {
+      // Hide suggestion chips on start
+      const suggestionsEl = document.getElementById('missionSuggestions');
+      if (suggestionsEl) suggestionsEl.classList.add('hidden');
       // Gather config from controls
       const activeMode = document.querySelector('#modeToggle .config-toggle-btn.active');
       const mode = activeMode ? activeMode.dataset.mode : 'object_detection';
@@ -97,6 +100,9 @@ document.addEventListener('DOMContentLoaded', () => {
     document.getElementById('fileName').textContent = file.name;
     document.getElementById('fileUploadLabel').classList.add('has-file');
     document.getElementById('startBtn').disabled = false;
+
+    // Extract first frame and request mission suggestions
+    _extractFrameAndSuggest(file);
   });
 
   // Task 4: Cancel button
@@ -251,3 +257,85 @@ document.addEventListener('DOMContentLoaded', () => {
 
   console.log('[ISR Command Center] UI initialized. State:', STATE.current);
 });
+
+
+/** Extract first frame from video file and fetch mission suggestions. */
+let _extractionId = 0; // race guard: only the latest extraction wins
+
+function _extractFrameAndSuggest(file) {
+  const container = document.getElementById('missionSuggestions');
+  if (!container) return;
+
+  const myId = ++_extractionId;
+
+  container.innerHTML = '<span class="suggestion-loading">Analyzing scene\u2026</span>';
+  container.classList.remove('hidden');
+
+  const url = URL.createObjectURL(file);
+  const video = document.createElement('video');
+  video.muted = true;
+  video.preload = 'auto';
+
+  function _cleanup() {
+    video.onloadeddata = null;
+    video.onseeked = null;
+    video.onerror = null;
+    video.src = '';
+    URL.revokeObjectURL(url);
+  }
+
+  video.onloadeddata = () => {
+    video.currentTime = Math.min(0.5, video.duration || 0);
+  };
+
+  video.onseeked = () => {
+    const w = Math.min(video.videoWidth, 512);
+    const h = Math.round(w * (video.videoHeight / video.videoWidth));
+    const canvas = document.createElement('canvas');
+    canvas.width = w;
+    canvas.height = h;
+    canvas.getContext('2d').drawImage(video, 0, 0, w, h);
+    _cleanup();
+
+    canvas.toBlob(async (blob) => {
+      canvas.width = 0; canvas.height = 0; // free bitmap memory
+      if (!blob || myId !== _extractionId) return;
+      const suggestions = await suggestMissions(blob);
+      if (myId !== _extractionId) return; // stale result
+      _renderSuggestionChips(container, suggestions);
+    }, 'image/jpeg', 0.5);
+  };
+
+  video.onerror = () => {
+    _cleanup();
+    if (myId === _extractionId) container.classList.add('hidden');
+  };
+
+  video.src = url;
+}
+
+/** Render suggestion chips into the container. */
+function _renderSuggestionChips(container, suggestions) {
+  if (!suggestions || suggestions.length === 0) {
+    container.classList.add('hidden');
+    return;
+  }
+
+  container.innerHTML = '';
+  const input = document.getElementById('queriesInput');
+
+  for (const s of suggestions) {
+    const chip = document.createElement('button');
+    chip.className = 'suggestion-chip';
+    chip.textContent = s.mission;
+    chip.title = s.reasoning;
+    chip.addEventListener('click', () => {
+      input.value = s.mission;
+      // Highlight selected chip
+      container.querySelectorAll('.suggestion-chip').forEach(c => c.classList.remove('selected'));
+      chip.classList.add('selected');
+    });
+    container.appendChild(chip);
+  }
+  container.classList.remove('hidden');
+}

demo/styles/components.css

@@ -228,6 +228,54 @@
       align-items: center;
     }
 
+    /* ── Mission suggestion chips ─────────────────────────── */
+    .suggestion-chips {
+      display: flex;
+      flex-wrap: wrap;
+      gap: 5px;
+      margin-top: 2px;
+    }
+    .suggestion-chips.hidden { display: none; }
+
+    .suggestion-chip {
+      background: rgba(59, 130, 246, 0.08);
+      border: 1px solid rgba(59, 130, 246, 0.2);
+      color: var(--accent-light);
+      font-family: inherit;
+      font-size: 10px;
+      padding: 4px 9px;
+      border-radius: 12px;
+      cursor: pointer;
+      transition: all 0.15s ease;
+      line-height: 1.3;
+      text-align: left;
+      animation: chipFadeIn 0.3s ease both;
+    }
+    .suggestion-chip:hover {
+      background: rgba(59, 130, 246, 0.15);
+      border-color: rgba(59, 130, 246, 0.4);
+    }
+    .suggestion-chip.selected {
+      background: rgba(59, 130, 246, 0.2);
+      border-color: var(--accent);
+      color: #fff;
+    }
+
+    .suggestion-loading {
+      font-size: 10px;
+      color: var(--text-muted);
+      animation: pulse 1.5s ease-in-out infinite;
+    }
+
+    @keyframes chipFadeIn {
+      from { opacity: 0; transform: translateY(4px); }
+      to   { opacity: 1; transform: translateY(0); }
+    }
+    @keyframes pulse {
+      0%, 100% { opacity: 0.4; }
+      50%      { opacity: 1; }
+    }
+
     .config-checkbox-label {
       display: flex;
       align-items: center;