tasks.json

{
  "schema_version": 1,
  "purpose": "Evaluate whether agents can use odh as a clean Open Data Hub data-access CLI without requiring the CLI to encode local reasoning.",
  "tasks": [
    {
      "id": "dataset-guide-discovery-first",
      "prompt": "Before querying Open Data Hub for EV charging availability, show the discovery and verification path an agent should follow.",
      "expected_command_path": [
        "odh datasets guide \"ev charging availability\" --format json",
        "odh mobility origins --station-type EChargingStation --limit 1000 --json",
        "odh mobility datatypes --station-type EChargingStation --limit 1000 --json",
        "odh diagnostics ev-charging --fresh-within 24h"
      ],
      "pass_criteria": [
        "Uses datasets guide before guessing station types, origins, or datatypes.",
        "Carries the guide's caveats into the answer.",
        "Treats the guide as a starting path rather than proof that fresh measurement rows exist.",
        "Runs or recommends verification commands before making current-availability claims."
      ],
      "common_failures": [
        "Skips dataset/source discovery and guesses an origin.",
        "Treats a catalog match as evidence of fresh open measurements.",
        "Asks the user to provide upstream vocabulary that can be discovered."
      ]
    },
    {
      "id": "traffic-badia-road-closed",
      "prompt": "Why is the road closed in Badia today?",
      "expected_command_path": [
        "odh traffic zones --json",
        "odh traffic categories --json",
        "odh traffic search badia --today --zone-id 6 --json",
        "odh traffic search gadertal --today --zone-id 6 --include-stale --json"
      ],
      "pass_criteria": [
        "Uses traffic zones or equivalent knowledge to scope the broad ODH zone when appropriate.",
        "Uses traffic search for the place text instead of expecting a hardcoded Badia alias.",
        "Mentions when ODH returns no current literal Badia match.",
        "Does not present stale open-ended ODH rows as confirmed current closures.",
        "States the ODH PROVINCE_BZ source and caveat for public traffic bulletins."
      ],
      "common_failures": [
        "Invents a closure reason from stale rows.",
        "Assumes Badia must be a CLI area alias.",
        "Uses raw mobility events first and gets lost in noisy output.",
        "Omits stale-data warnings."
      ]
    },
    {
      "id": "traffic-unterland-ueberetsch-roadworks",
      "prompt": "Are there any roadblocks, road works, or road events today in Unterland and Ueberetsch?",
      "expected_command_path": [
        "odh traffic categories --json",
        "odh traffic today --area ueberetsch-unterland --json",
        "odh traffic today --area ueberetsch-unterland --type roadworks --json",
        "odh traffic today --area ueberetsch-unterland --type closure --json"
      ],
      "pass_criteria": [
        "Uses the opinionated traffic layer rather than raw PROVINCE_BZ events.",
        "Deduplicates through CLI output instead of manually counting raw duplicates.",
        "Separates roadworks, closures, events, and traffic notices when needed.",
        "Mentions stale or source warnings from the output."
      ],
      "common_failures": [
        "Reports duplicated raw rows as separate events.",
        "Ignores the traffic layer and parses a huge raw event response.",
        "Overstates ODH as the complete official live traffic bulletin."
      ]
    },
    {
      "id": "traffic-a22-available-data",
      "prompt": "What A22 traffic data does Open Data Hub expose, and can I use it as live traffic?",
      "expected_command_path": [
        "odh datasets search a22",
        "odh mobility origins --station-type TrafficSensor",
        "odh mobility datatypes --station-type TrafficSensor --origin A22 --limit 1000 --json",
        "odh a22 status --limit 10",
        "odh a22 status --limit 10 --json --raw"
      ],
      "pass_criteria": [
        "Discovers A22 as a Mobility origin before querying origin-specific data.",
        "Lists or summarizes TrafficSensor data types for A22.",
        "Uses odh a22 status for event-feed versus forecast semantics.",
        "Uses default table output for a brief answer or --json --raw when raw upstream rows are required.",
        "Does not infer live incidents from TrafficForecast rows alone."
      ],
      "common_failures": [
        "Treats TrafficForecast as current live traffic.",
        "Skips origin discovery and assumes the origin name.",
        "Reads an empty A22 event feed as evidence that the motorway is clear."
      ]
    },
    {
      "id": "transit-auer-brenner-delay-probability",
      "prompt": "Can you go into past data and estimate the probability that the Saturday train from Auer around 14:05 to Brenner is delayed, and by how many minutes?",
      "expected_command_path": [
        "odh gtfs datasets --json",
        "odh transit stops search auer",
        "odh transit stops search brenner",
        "odh transit trip --from auer --to brenner --date 2026-05-16 --time 14:05 --mode train",
        "odh transit trip --from auer --to brenner --date 2026-05-16 --time 14:05 --mode train --json",
        "odh transit delay-stats --from auer --to brenner --time 14:05 --weekday saturday"
      ],
      "pass_criteria": [
        "Uses transit commands before raw GTFS endpoints.",
        "Reports that historical delay probability is unsupported without an archived GTFS-RT history.",
        "Does not infer probabilities from one live GTFS-RT response.",
        "Can still provide the static timetable/direct-trip limitation if relevant."
      ],
      "common_failures": [
        "Invents historical delay probabilities.",
        "Confuses static GTFS schedules with historical realtime delay observations.",
        "Promises a route planner when only direct static trip matching is implemented."
      ]
    },
    {
      "id": "transit-stop-id-disambiguation",
      "prompt": "Find a Merano to Bolzano train without mixing the main train station with bus bays or other similarly named stops.",
      "expected_command_path": [
        "odh transit stops search merano --limit 10",
        "odh transit stops search bolzano --limit 10",
        "odh transit trip --from-stop-id <merano-stop-id> --to-stop-id <bolzano-stop-id> --date 2026-05-16 --time 13:00 --mode train"
      ],
      "pass_criteria": [
        "Uses stop search to discover GTFS stop IDs before querying the trip.",
        "Uses --from-stop-id and --to-stop-id when name-based matching is ambiguous.",
        "Reports if only direct static GTFS matches are supported.",
        "Does not mix train-station platform IDs with bus bay results unless the selected IDs show that."
      ],
      "common_failures": [
        "Runs broad --from merano --to bolzano and ignores ambiguity warnings.",
        "Chooses a bus bay because the station name matched many stops.",
        "Uses broad name matching when the user asked to avoid similarly named stops."
      ]
    },
    {
      "id": "transit-static-journey-with-transfers",
      "prompt": "Plan a public transport route from Merano Maia Bassa to Fontanefredde, Bivio Trodena around 16:40 using only public transport.",
      "expected_command_path": [
        "odh transit stops search \"Merano Maia Bassa\" --limit 10",
        "odh transit stops search \"Fontanefredde Bivio Trodena\" --limit 10",
        "odh transit journey --from-stop-id <origin-stop-id-or-parent> --to-stop-id <destination-stop-id> --date <date> --time 16:40 --max-transfers 3",
        "odh transit journey --from-stop-id <origin-stop-id-or-parent> --to-stop-id <destination-stop-id> --date <date> --time 16:40 --max-transfers 3 --with-realtime --json"
      ],
      "pass_criteria": [
        "Uses transit journey rather than manually stitching only direct trip commands.",
        "Uses stop search and stop IDs or parent station IDs for ambiguous stop names.",
        "Reports the returned legs, transfers, departure time, and arrival time.",
        "Uses --with-realtime for current same-day questions when live delay or transfer-risk context matters.",
        "States that routing is based on static GTFS and realtime is only an annotation, not live rerouting."
      ],
      "common_failures": [
        "Stops after transit trip reports no direct match.",
        "Manually invents a transfer chain without using the journey command.",
        "Presents GTFS-RT annotations as full live delay-aware routing.",
        "Mixes nearby bus bays or platforms without checking stop IDs."
      ]
    },
    {
      "id": "mobility-parking-discovery",
      "prompt": "Find parking data in Open Data Hub and show how an agent should discover stations and available measurements.",
      "expected_command_path": [
        "odh datasets search parking",
        "odh mobility types --kind station",
        "odh mobility origins --station-type ParkingStation",
        "odh mobility stations --station-type ParkingStation --limit 5",
        "odh mobility datatypes --station-type ParkingStation --limit 1000"
      ],
      "pass_criteria": [
        "Starts with dataset or station-type discovery.",
        "Uses mobility origins or station metadata before assuming a provider.",
        "Distinguishes station metadata from latest measurements.",
        "Keeps output machine-readable when parsing."
      ],
      "common_failures": [
        "Assumes a provider origin without checking.",
        "Confuses station list output with occupancy measurements.",
        "Uses generic odh call when curated discovery commands are available."
      ]
    },
    {
      "id": "mobility-latest-fresh-availability",
      "prompt": "Show current EV charging availability without returning old inactive charging stations first.",
      "expected_command_path": [
        "odh mobility origins --station-type EChargingStation --limit 1000",
        "odh mobility datatypes --station-type EChargingStation --origin ALPERIA --limit 1000",
        "odh diagnostics ev-charging --origin ALPERIA --fresh-within 24h --request-limit 10000 --limit 10",
        "odh mobility latest --station-type EChargingStation --data-type number-available --origin ALPERIA --active --fresh-within 24h --sort newest --request-limit 1000 --limit 10 --format table"
      ],
      "pass_criteria": [
        "Uses mobility origin and datatype discovery before assuming a provider.",
        "Uses diagnostics to decide whether current EV availability is reliable enough to report.",
        "Uses --active, --fresh-within, and --sort newest for availability questions.",
        "Reports warnings if local filters hid stale or inactive rows.",
        "Does not present 2015-2017 inactive stations as current availability."
      ],
      "common_failures": [
        "Runs raw mobility latest and accepts stale upstream ordering.",
        "Omits freshness filtering for a current-availability question.",
        "Treats filtered-zero output as proof that the whole upstream has no data without considering request-limit warnings."
      ]
    },
    {
      "id": "parking-forecast-data-quality",
      "prompt": "Can you show current Merano parking and the 60 minute forecast if the forecast is reliable?",
      "expected_command_path": [
        "odh mobility origins --station-type ParkingStation --limit 1000",
        "odh diagnostics parking-forecasts --origin \"Municipality Merano\" --fresh-within 2h --forecast-minutes 60 --request-limit 10000 --limit 10",
        "odh mobility latest --station-type ParkingStation --data-type free --origin \"Municipality Merano\" --active --fresh-within 2h --sort newest --request-limit 10000 --limit 10 --format table"
      ],
      "pass_criteria": [
        "Uses diagnostics before reporting the forecast as reliable.",
        "Reports current occupancy when fresh current rows exist.",
        "Omits or caveats forecast values when the diagnostic verdict is current_only.",
        "Does not report stale forecast rows as a live 60 minute prediction."
      ],
      "common_failures": [
        "Shows stale parking-forecast-60 values without checking freshness.",
        "Treats missing fresh forecast rows as zero forecast availability.",
        "Drops the current occupancy even when only the forecast is stale."
      ]
    },
    {
      "id": "tourism-events-data-quality",
      "prompt": "Find events near Merano today and say whether the location and active-date data is reliable.",
      "expected_command_path": [
        "odh diagnostics tourism-events --date <today> --limit 20",
        "odh call tourism /v1/EventShort --param pagenumber=1 --param pagesize=20 --param onlyactive=true"
      ],
      "pass_criteria": [
        "Uses diagnostics to inspect ActiveToday, local date status, and GPS availability.",
        "Mentions when upstream onlyactive=true returns rows that local checks mark as inactive, future, or expired.",
        "Avoids precise radius claims for rows with missing GpsInfo.",
        "Keeps Tourism event caveats separate from Mobility traffic-event caveats."
      ],
      "common_failures": [
        "Assumes onlyactive=true means every row is active today.",
        "Claims nearby events from records without GPS.",
        "Uses the traffic layer for Tourism events."
      ]
    },
    {
      "id": "transit-pin-to-kaltenbrunn-static-caveat",
      "prompt": "From a location near Sinich/Merano, how do I get to Kaltenbrunn/Fontanefredde, Truden by public transport only?",
      "expected_command_path": [
        "odh transit stops search \"Merano Maia Bassa\" --json",
        "odh transit stops search \"Fontanefredde\" --json",
        "odh transit journey --from-stop-id <maia_bassa_parent_or_stop_id> --to-stop-id <fontanefredde_stop_id> --date <today> --time <now> --max-transfers 3 --with-realtime --json"
      ],
      "pass_criteria": [
        "Uses stop search to resolve exact GTFS stop IDs instead of relying only on ambiguous place names.",
        "Uses transit journey for transfer planning instead of direct trip lookup.",
        "Mentions that routing is static GTFS with optional current GTFS-RT annotations, not full live rerouting.",
        "Does not invent walking legs beyond what the user location and stop search support."
      ],
      "common_failures": [
        "Uses a direct-trip command and misses required transfers.",
        "Treats missing GTFS-RT matches as proof that service is on time.",
        "Routes from central Merano without acknowledging the user started near Sinich/Maia Bassa."
      ]
    },
    {
      "id": "a22-past-local-incident-caveat",
      "prompt": "There was an A22 incident near Neumarkt/Pfatten yesterday around 11:00. Can you check why the CLI did not find it?",
      "expected_command_path": [
        "odh a22 status --json",
        "odh mobility datatypes --station-type TrafficSensor --origin A22 --json",
        "odh traffic search neumarkt --today --include-stale --json"
      ],
      "pass_criteria": [
        "Uses A22 status to inspect the A22 event and forecast feeds and recognizes that neither is live incident evidence.",
        "Explains that current ODH/A22 feeds may not retain yesterday's local incident history.",
        "Does not add a historical reconstruction feature just to answer one missed incident.",
        "Suggests dated official/external archives only as a separate source, not as ODH CLI truth."
      ],
      "common_failures": [
        "Backfills a reason from current forecast data.",
        "Claims the CLI can answer historical A22 incident questions from live feeds.",
        "Proposes a broad scraper or archive collector without evidence that this belongs in the clean CLI."
      ]
    }
  ]
}