Skip to content

Commit 5a33f8b

Browse files
devin-ai-integration[bot]thymikee
devin-ai-integration[bot]
and
thymikee
authored
feat(maestro): numeric option fields accept ${VAR} lookup interpolation (#1293) (#1342)
* feat(maestro): numeric option fields accept ${VAR} lookup interpolation (#1293) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(maestro): validate resolved numeric strings before coercion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(maestro): address review feedback on numeric resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(maestro-conformance): preserve unresolved ${VAR} numeric tokens in canonical model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Michał Pierzchała <thymikee@gmail.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent a924f0b commit 5a33f8b

25 files changed

Lines changed: 801 additions & 169 deletions

‎scripts/maestro-conformance/build-manifest.mjs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ const NOTES = {
2828
'authored/extended-wait': 'Coverage: extendedWaitUntil (upstream 042 interpolates ${TIMEOUT} from a flow env block).',
2929
'authored/repeat': 'Coverage: repeat.times (upstream 053 also uses the unsupported evalScript command).',
3030
'authored/presskey': 'Coverage: pressKey supported keys (upstream 034 exercises many unsupported keycodes).',
31+
'authored/numeric-variable-tap': 'Coverage: tapOn repeat/delay/index numeric option fields accept ${VAR} tokens (upstream rejects repeat/delay as integer-typed; agent-device is lenient).',
32+
'authored/numeric-variable-doubletap': 'Coverage: doubleTapOn delay accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient).',
33+
'authored/numeric-variable-swipe': 'Coverage: swipe duration accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient).',
34+
'authored/numeric-variable-erase': 'Coverage: eraseText charactersToErase accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient).',
35+
'authored/numeric-variable-tap-index': 'Coverage: tapOn index accepts a ${VAR} token and projects identically through the canonical model.',
36+
'authored/numeric-variable-wait': 'Coverage: waitForAnimationToEnd timeout accepts a ${VAR} token and projects identically through the canonical model.',
3137
'invalid/bad-swipe-direction': 'Lenient-guard: unknown SwipeDirection enum value.',
3238
'invalid/unknown-command': 'Lenient-guard: unknown command name (tapOn typo).',
3339
'invalid/malformed-selector': 'Lenient-guard: selector given as a sequence.',

‎scripts/maestro-conformance/corpus/authored/numeric-variable-doubletap.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: com.example.app
2+
---
3+
- doubleTapOn:
4+
text: Button
5+
delay: ${DELAY}

‎scripts/maestro-conformance/corpus/authored/numeric-variable-erase.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
appId: com.example.app
2+
---
3+
- eraseText: ${COUNT}

‎scripts/maestro-conformance/corpus/authored/numeric-variable-swipe.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: com.example.app
2+
---
3+
- swipe:
4+
direction: DOWN
5+
duration: ${DURATION}

‎scripts/maestro-conformance/corpus/authored/numeric-variable-tap-index.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: com.example.app
2+
---
3+
- tapOn:
4+
text: Button
5+
index: ${INDEX}

‎scripts/maestro-conformance/corpus/authored/numeric-variable-tap.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
appId: com.example.app
2+
---
3+
- tapOn:
4+
text: Button
5+
index: ${INDEX}
6+
repeat: ${REPEAT}
7+
delay: ${DELAY}

‎scripts/maestro-conformance/corpus/authored/numeric-variable-wait.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
appId: com.example.app
2+
---
3+
- waitForAnimationToEnd:
4+
timeout: ${TIMEOUT}

‎scripts/maestro-conformance/corpus/manifest.json‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,54 @@
480480
"note": "Coverage: extendedWaitUntil (upstream 042 interpolates ${TIMEOUT} from a flow env block)."
481481
}
482482
},
483+
{
484+
"id": "authored/numeric-variable-doubletap",
485+
"file": "authored/numeric-variable-doubletap.yaml",
486+
"origin": {
487+
"kind": "authored",
488+
"note": "Coverage: doubleTapOn delay accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient)."
489+
}
490+
},
491+
{
492+
"id": "authored/numeric-variable-erase",
493+
"file": "authored/numeric-variable-erase.yaml",
494+
"origin": {
495+
"kind": "authored",
496+
"note": "Coverage: eraseText charactersToErase accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient)."
497+
}
498+
},
499+
{
500+
"id": "authored/numeric-variable-swipe",
501+
"file": "authored/numeric-variable-swipe.yaml",
502+
"origin": {
503+
"kind": "authored",
504+
"note": "Coverage: swipe duration accepts a ${VAR} token (upstream rejects as integer-typed; agent-device is lenient)."
505+
}
506+
},
507+
{
508+
"id": "authored/numeric-variable-tap-index",
509+
"file": "authored/numeric-variable-tap-index.yaml",
510+
"origin": {
511+
"kind": "authored",
512+
"note": "Coverage: tapOn index accepts a ${VAR} token and projects identically through the canonical model."
513+
}
514+
},
515+
{
516+
"id": "authored/numeric-variable-tap",
517+
"file": "authored/numeric-variable-tap.yaml",
518+
"origin": {
519+
"kind": "authored",
520+
"note": "Coverage: tapOn repeat/delay/index numeric option fields accept ${VAR} tokens (upstream rejects repeat/delay as integer-typed; agent-device is lenient)."
521+
}
522+
},
523+
{
524+
"id": "authored/numeric-variable-wait",
525+
"file": "authored/numeric-variable-wait.yaml",
526+
"origin": {
527+
"kind": "authored",
528+
"note": "Coverage: waitForAnimationToEnd timeout accepts a ${VAR} token and projects identically through the canonical model."
529+
}
530+
},
483531
{
484532
"id": "authored/presskey",
485533
"file": "authored/presskey.yaml",

‎scripts/maestro-conformance/expected-divergence.ts‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
export type FlowDivergence = {
66
/** Expected non-identical classification for this corpus flow. */
7-
classification: 'we-reject' | 'mismatch';
7+
classification: 'we-reject' | 'mismatch' | 'we-are-lenient';
88
/** Why the divergence is intentional. */
99
reason: string;
1010
/** Upstream commands/options our engine deliberately does not support. */
@@ -13,10 +13,11 @@ export type FlowDivergence = {
1313
tracking?: string;
1414
};
1515

16-
// Keyed by corpus flow id. Every flow the verifier classifies as `we-reject` or
17-
// `mismatch` must appear here; anything undeclared fails the suite. This is the
18-
// mechanical parity backlog — the list the old hand-typed fixture could not
19-
// produce. `tracking` points at the Maestro compat tracker for option-level gaps.
16+
// Keyed by corpus flow id. Every flow the verifier classifies as `we-reject`,
17+
// `mismatch`, or `we-are-lenient` must appear here; anything undeclared fails the
18+
// suite. This is the mechanical parity backlog — the list the old hand-typed
19+
// fixture could not produce. `tracking` points at the Maestro compat tracker for
20+
// option-level gaps.
2021
const COMPAT_TRACKER = 'https://github.com/callstack/agent-device/issues/558';
2122

2223
export const FLOW_DIVERGENCES: Record<string, FlowDivergence> = {
@@ -73,22 +74,17 @@ export const FLOW_DIVERGENCES: Record<string, FlowDivergence> = {
7374
// --- Unsupported options on supported commands (parity gaps) ---
7475
'upstream/032_element_index': {
7576
classification: 'we-reject',
76-
reason: 'A literal tapOn.index is supported; the flow also uses a ${0 + 1} JS-expression index.',
77-
unsupported: ['tapOn.index (expression)'],
78-
tracking: COMPAT_TRACKER,
77+
reason:
78+
'tapOn.index supports ${VAR} lookup; the flow uses a ${0 + 1} JS expression, which is not supported.',
79+
unsupported: ['tapOn.index (JS expression)'],
80+
tracking: 'https://github.com/callstack/agent-device/issues/1292',
7981
},
8082
'upstream/034_press_key': {
8183
classification: 'we-reject',
8284
reason: 'pressKey supports back/enter/home/return; the flow exercises ~30 Android/TV keycodes.',
8385
unsupported: ['pressKey (extended keycodes)'],
8486
tracking: COMPAT_TRACKER,
8587
},
86-
'upstream/042_extended_wait': {
87-
classification: 'we-reject',
88-
reason: 'A literal extendedWaitUntil.timeout is supported; the flow interpolates ${TIMEOUT} from a flow env block (unresolved ${} is fail-loud).',
89-
unsupported: ['extendedWaitUntil.timeout (interpolation)'],
90-
tracking: COMPAT_TRACKER,
91-
},
9288
'upstream/076_optional_assertion': {
9389
classification: 'we-reject',
9490
reason: 'assertTrue is outside the supported subset; optional is now supported on scrollUntilVisible and extendedWaitUntil.',
@@ -113,6 +109,31 @@ export const FLOW_DIVERGENCES: Record<string, FlowDivergence> = {
113109
unsupported: ['tapOn.waitToSettleTimeoutMs'],
114110
tracking: COMPAT_TRACKER,
115111
},
112+
// --- agent-device supports ${VAR} in integer-typed numeric option fields that the pinned upstream parser rejects ---
113+
'authored/numeric-variable-tap': {
114+
classification: 'we-are-lenient',
115+
reason:
116+
'agent-device supports ${VAR} lookup in tapOn.index/repeat/delay; the pinned upstream Maestro 2.5.1 parser only accepts ${VAR} in tapOn.index (string-typed) and rejects repeat/delay as integer-typed.',
117+
tracking: 'https://github.com/callstack/agent-device/issues/1293',
118+
},
119+
'authored/numeric-variable-doubletap': {
120+
classification: 'we-are-lenient',
121+
reason:
122+
'agent-device supports ${VAR} lookup in doubleTapOn.delay; the pinned upstream Maestro 2.5.1 parser rejects it as integer-typed.',
123+
tracking: 'https://github.com/callstack/agent-device/issues/1293',
124+
},
125+
'authored/numeric-variable-swipe': {
126+
classification: 'we-are-lenient',
127+
reason:
128+
'agent-device supports ${VAR} lookup in swipe.duration; the pinned upstream Maestro 2.5.1 parser rejects it as integer-typed.',
129+
tracking: 'https://github.com/callstack/agent-device/issues/1293',
130+
},
131+
'authored/numeric-variable-erase': {
132+
classification: 'we-are-lenient',
133+
reason:
134+
'agent-device supports ${VAR} lookup in eraseText.charactersToErase; the pinned upstream Maestro 2.5.1 parser rejects it as integer-typed.',
135+
tracking: 'https://github.com/callstack/agent-device/issues/1293',
136+
},
116137
};
117138

118139
// Layer-2 semantic vectors that describe an upstream constant we intentionally

‎scripts/maestro-conformance/fixtures/layer1-parser.json‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3369,6 +3369,129 @@
33693369
}
33703370
]
33713371
},
3372+
{
3373+
"id": "authored/numeric-variable-doubletap",
3374+
"file": "authored/numeric-variable-doubletap.yaml",
3375+
"status": "rejected",
3376+
"error": {
3377+
"class": "FlowParseException",
3378+
"message": "Incorrect Format: delay"
3379+
}
3380+
},
3381+
{
3382+
"id": "authored/numeric-variable-erase",
3383+
"file": "authored/numeric-variable-erase.yaml",
3384+
"status": "rejected",
3385+
"error": {
3386+
"class": "SyntaxError",
3387+
"message": "Failed to parse file: <corpus>/authored/numeric-variable-erase.yaml"
3388+
}
3389+
},
3390+
{
3391+
"id": "authored/numeric-variable-swipe",
3392+
"file": "authored/numeric-variable-swipe.yaml",
3393+
"status": "rejected",
3394+
"error": {
3395+
"class": "FlowParseException",
3396+
"message": "Parsing Failed"
3397+
}
3398+
},
3399+
{
3400+
"id": "authored/numeric-variable-tap-index",
3401+
"file": "authored/numeric-variable-tap-index.yaml",
3402+
"status": "parsed",
3403+
"commands": [
3404+
{
3405+
"type": "ApplyConfigurationCommand",
3406+
"fields": {
3407+
"config": {
3408+
"appId": "com.example.app",
3409+
"name": null,
3410+
"tags": [],
3411+
"ext": {},
3412+
"onFlowStart": null,
3413+
"onFlowComplete": null,
3414+
"properties": {}
3415+
},
3416+
"label": null,
3417+
"optional": false
3418+
}
3419+
},
3420+
{
3421+
"type": "TapOnElementCommand",
3422+
"fields": {
3423+
"selector": {
3424+
"textRegex": "Button",
3425+
"idRegex": null,
3426+
"size": null,
3427+
"below": null,
3428+
"above": null,
3429+
"leftOf": null,
3430+
"rightOf": null,
3431+
"containsChild": null,
3432+
"containsDescendants": null,
3433+
"traits": null,
3434+
"index": "${INDEX}",
3435+
"enabled": null,
3436+
"optional": false,
3437+
"selected": null,
3438+
"checked": null,
3439+
"focused": null,
3440+
"childOf": null,
3441+
"css": null
3442+
},
3443+
"retryIfNoChange": false,
3444+
"waitUntilVisible": false,
3445+
"longPress": false,
3446+
"repeat": null,
3447+
"waitToSettleTimeoutMs": null,
3448+
"relativePoint": null,
3449+
"label": null,
3450+
"optional": false
3451+
}
3452+
}
3453+
]
3454+
},
3455+
{
3456+
"id": "authored/numeric-variable-tap",
3457+
"file": "authored/numeric-variable-tap.yaml",
3458+
"status": "rejected",
3459+
"error": {
3460+
"class": "FlowParseException",
3461+
"message": "Incorrect Format: repeat"
3462+
}
3463+
},
3464+
{
3465+
"id": "authored/numeric-variable-wait",
3466+
"file": "authored/numeric-variable-wait.yaml",
3467+
"status": "parsed",
3468+
"commands": [
3469+
{
3470+
"type": "ApplyConfigurationCommand",
3471+
"fields": {
3472+
"config": {
3473+
"appId": "com.example.app",
3474+
"name": null,
3475+
"tags": [],
3476+
"ext": {},
3477+
"onFlowStart": null,
3478+
"onFlowComplete": null,
3479+
"properties": {}
3480+
},
3481+
"label": null,
3482+
"optional": false
3483+
}
3484+
},
3485+
{
3486+
"type": "WaitForAnimationToEndCommand",
3487+
"fields": {
3488+
"timeout": "${TIMEOUT}",
3489+
"label": null,
3490+
"optional": false
3491+
}
3492+
}
3493+
]
3494+
},
33723495
{
33733496
"id": "authored/presskey",
33743497
"file": "authored/presskey.yaml",
@@ -3919,5 +4042,5 @@
39194042
]
39204043
}
39214044
],
3922-
"contentHash": "7e657d22c33fd784980ca793962fce001d824f2fa9efdc65cf3c95174138b19a"
4045+
"contentHash": "b81dbe35d8b2b95c6c0c0b9f49514c00a0729f618da693574ef84b6feb05a5b4"
39234046
}

0 commit comments

Comments
 (0)