diff --git a/md/system_prompt.md b/md/system_prompt.md index 7a1dfdde..d5d2bb81 100644 --- a/md/system_prompt.md +++ b/md/system_prompt.md @@ -61,9 +61,23 @@ Do not name specific members of a set — not even as examples. Describe the gov ### Spec Format -For each extracted function file (for example, `calculate_average.py`), write TWO -separate JSON files in the SAME directory. Do NOT modify the original function -source file. +For each extracted function file, write TWO separate JSON files in the SAME +directory. Do NOT modify the original function source file. + +**CRITICAL — filename rule:** The output filename MUST be **exactly** the +function file name **including its source extension**, with `.spec.json` or +`.info.json` appended. The source extension (.rs, .cpp, .py, .c, etc.) is +part of the filename and MUST NOT be removed, changed, or normalized. The +same naming rule applies to both `.spec.json` and `.info.json`. + +**Examples:** + +| Function file | Spec output | Info output | +|---|---|---| +| `Preprocessor::preprocess_source.rs` | `Preprocessor::preprocess_source.rs.spec.json` | `Preprocessor::preprocess_source.rs.info.json` | +| `calculate_average.py` | `calculate_average.py.spec.json` | `calculate_average.py.info.json` | +| `LocalStorage::Flush.cpp` | `LocalStorage::Flush.cpp.spec.json` | `LocalStorage::Flush.cpp.info.json` | +| `src/lib/parser.c` | `src/lib/parser.c.spec.json` | `src/lib/parser.c.info.json` | **`.spec.json`** — the function's own behavioral specification: diff --git a/md/workflow_spec_step4_batch.md b/md/workflow_spec_step4_batch.md index 3bd8c809..dd407c77 100644 --- a/md/workflow_spec_step4_batch.md +++ b/md/workflow_spec_step4_batch.md @@ -12,17 +12,31 @@ You are given a single batch prompt file path in the prompt. Your ONLY job is to 4. For EACH function listed in the batch prompt: a. Read the extracted function file b. If layer > 0, read earlier-layer caller specs mentioned in the batch prompt - c. Generate a behavioral spec and write it to `.spec.json` in the same directory - d. Generate callee expectations and write them to `.info.json` in the same directory + c. Generate a behavioral spec and write it to `.spec.json` — the filename MUST include the source extension (see Spec Format below) + d. Generate callee expectations and write them to `.info.json` — the filename MUST include the source extension (see Spec Format below) e. Do NOT modify the original function source file --- ## Spec Format -For each extracted function file (for example, `calculate_average.py`), write TWO -separate JSON files in the SAME directory. Do NOT modify the original function -source file. +For each extracted function file, write TWO separate JSON files in the SAME +directory. Do NOT modify the original function source file. + +**CRITICAL — filename rule:** The output filename MUST be **exactly** the +function file name **including its source extension**, with `.spec.json` or +`.info.json` appended. The source extension (.rs, .cpp, .py, .c, etc.) is +part of the filename and MUST NOT be removed, changed, or normalized. The +same naming rule applies to both `.spec.json` and `.info.json`. + +**Examples:** + +| Function file | Spec output | Info output | +|---|---|---| +| `Preprocessor::preprocess_source.rs` | `Preprocessor::preprocess_source.rs.spec.json` | `Preprocessor::preprocess_source.rs.info.json` | +| `calculate_average.py` | `calculate_average.py.spec.json` | `calculate_average.py.info.json` | +| `LocalStorage::Flush.cpp` | `LocalStorage::Flush.cpp.spec.json` | `LocalStorage::Flush.cpp.info.json` | +| `src/lib/parser.c` | `src/lib/parser.c.spec.json` | `src/lib/parser.c.info.json` | **`.spec.json`** — the function's own behavioral specification: diff --git a/src/file_utils.py b/src/file_utils.py index 80b8984c..11de7986 100644 --- a/src/file_utils.py +++ b/src/file_utils.py @@ -1,3 +1,4 @@ +import logging import json import os import re @@ -118,6 +119,163 @@ def is_file_ready(file_path): return _is_valid_spec_json(spec) and _is_valid_info_json(info) +def _is_valid_sidecar_file(path): + """Return True if *path* is a sidecar JSON file with valid FM-Agent schema.""" + if not os.path.isfile(path): + return False + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return False + if path.endswith(".spec.json"): + return _is_valid_spec_json(data) + if path.endswith(".info.json"): + return _is_valid_info_json(data) + return False + + +def normalize_spec_filenames(function_files, all_function_files=None): + """Fix sidecar filenames where the LLM dropped the source extension. + + For each function file ``foo.rs`` the pipeline expects sidecars + ``foo.rs.spec.json`` and ``foo.rs.info.json``. Some LLMs instead + produce ``foo.spec.json`` and ``foo.info.json``. + + When the mapping is unambiguous and both bare sidecars form a valid + pair, this function replaces both expected sidecars together. Treating + the pair as one unit avoids combining a sidecar from an earlier attempt + with one from the latest attempt. If a bare name has multiple possible + owners, both bare sidecars are removed and the canonical pairs remain + unready for the caller's bounded retry loop. An incomplete canonical pair + is never completed from bare sidecars in the same watcher invocation: all + four files are cleared so a reader that already loaded the old canonical + half cannot combine it with a newly published half. + + Args: + function_files: Function paths whose sidecars may be normalized. + all_function_files: Optional complete scope used to detect ambiguous bare + names. This lets one completed batch normalize its own outputs while + still checking collisions against the rest of the layer. + + Returns: + list[str]: Function paths whose complete sidecar pair was replaced. + """ + if not function_files: + return [] + + function_files = list(dict.fromkeys(function_files)) + if all_function_files is None: + all_function_files = function_files + else: + all_function_files = list(dict.fromkeys( + [*all_function_files, *function_files] + )) + normalized = [] + + base_map = {} + for func_path in all_function_files: + base = os.path.splitext(func_path)[0] + base_map.setdefault(base, []).append(func_path) + + ambiguous_bases = { + base for base, candidates in base_map.items() if len(candidates) > 1 + } + for base in sorted({os.path.splitext(path)[0] for path in function_files}): + if base not in ambiguous_bases: + continue + + removed = [] + for suffix in (".spec.json", ".info.json"): + alternate_path = f"{base}{suffix}" + try: + os.remove(alternate_path) + removed.append(alternate_path) + except FileNotFoundError: + pass + + if removed: + logging.warning( + "Removed ambiguous extension-dropped sidecar file(s) %s; " + "they could belong to multiple source files: %s. Canonical " + "sidecars remain pending for retry.", + removed, + base_map[base], + ) + + for func_path in function_files: + base = os.path.splitext(func_path)[0] + if base in ambiguous_bases: + # A bare sidecar can never be safely assigned to one of these + # functions. Leave every incomplete canonical pair unready so the + # existing bounded retry loop asks the producer to regenerate it. + continue + + expected_spec = f"{func_path}.spec.json" + expected_info = f"{func_path}.info.json" + + expected_spec_exists = os.path.isfile(expected_spec) + expected_info_exists = os.path.isfile(expected_info) + expected_spec_valid = _is_valid_sidecar_file(expected_spec) + expected_info_valid = _is_valid_sidecar_file(expected_info) + if expected_spec_valid and expected_info_valid: + continue + + alt_spec = f"{base}.spec.json" + alt_info = f"{base}.info.json" + if not (_is_valid_sidecar_file(alt_spec) + and _is_valid_sidecar_file(alt_info)): + if os.path.isfile(alt_spec) or os.path.isfile(alt_info): + logging.warning( + "Not normalizing incomplete or invalid sidecar pair for %s; " + "it will be retried.", + func_path, + ) + continue + + if expected_spec_exists or expected_info_exists: + # A concurrent readiness check may already have loaded the valid + # half of this incomplete canonical pair. Publishing the missing + # half now could make that reader accept an old/new pair. Remove + # both canonical and bare pairs, then let the next attempt generate + # a complete pair from a clean state before its watcher starts. + removed = [] + for path in (expected_spec, expected_info, alt_spec, alt_info): + try: + os.remove(path) + removed.append(path) + except FileNotFoundError: + pass + logging.warning( + "Removed conflicting canonical and extension-dropped sidecars " + "%s for %s; the complete pair remains pending for retry.", + removed, + func_path, + ) + continue + + # The retry coordinator removes incomplete canonical pairs before it + # starts producers and the streaming watcher. Publish info first and + # spec last so the canonical spec path acts as the readiness gate: once + # it becomes readable, both sidecars come from this attempt. + for alt, expected in ( + (alt_info, expected_info), + (alt_spec, expected_spec), + ): + os.replace(alt, expected) + + normalized.append(func_path) + logging.info( + "Normalized sidecar pair: (%s, %s) -> (%s, %s)", + alt_spec, + alt_info, + expected_spec, + expected_info, + ) + + return normalized + + # Directories that typically contain test code _TEST_DIR_NAMES = { "test", "tests", "__tests__", "testing", "test_helpers", diff --git a/src/generate_batch_prompts.py b/src/generate_batch_prompts.py index 680d35a1..f42b1fcc 100644 --- a/src/generate_batch_prompts.py +++ b/src/generate_batch_prompts.py @@ -333,13 +333,24 @@ def build_prompt( lines.append("## SPEC FORMAT (write JSON files; do NOT modify source files)") lines.append("") lines.append( - "For each function file ``, " - "write TWO JSON files in the SAME directory. " - "`` includes its original extension " - "(for example, `foo.py` must produce `foo.py.spec.json` " - "and `foo.py.info.json`):" + "For each function file, write TWO JSON files in the SAME directory." ) lines.append("") + lines.append( + "CRITICAL: The output filename MUST be exactly the function file name " + "INCLUDING its source extension (.rs, .cpp, .py, .c, etc.), with " + ".spec.json or .info.json appended. Do NOT remove, change, or " + "normalize the source file extension." + ) + lines.append("") + lines.append("Examples:") + lines.append(" Preprocessor::preprocess_source.rs -> Preprocessor::preprocess_source.rs.spec.json") + lines.append(" Preprocessor::preprocess_source.rs.info.json") + lines.append(" calculate_average.py -> calculate_average.py.spec.json") + lines.append(" calculate_average.py.info.json") + lines.append(" LocalStorage::Flush.cpp -> LocalStorage::Flush.cpp.spec.json") + lines.append(" LocalStorage::Flush.cpp.info.json") + lines.append("") lines.append("`.spec.json`:") lines.append("```json") lines.append( diff --git a/src/spec_generation_and_verification.py b/src/spec_generation_and_verification.py index fecf2898..f68043af 100644 --- a/src/spec_generation_and_verification.py +++ b/src/spec_generation_and_verification.py @@ -12,7 +12,12 @@ from config import MAX_WORKERS, OPENCODE_MAX_RETRIES, OPENCODE_SPEC_MODEL from src.domain_knowledge import list_staged_domain_knowledge_relpaths -from src.file_utils import _get_incomplete_verification_files, _get_phase_files, is_file_ready +from src.file_utils import ( + _get_incomplete_verification_files, + _get_phase_files, + is_file_ready, + normalize_spec_filenames, +) from src.generate_topdown_layers import generate_topdown_layers from src.llm_client import build_llm_cli_command from src.opencode_trace import function_id_from_extracted_path, run_opencode_traced @@ -31,6 +36,56 @@ def _get_pending_batches(batches, proj_dir): return pending +def _invalidate_verification_artifacts(file_list, output_dir, work_dir): + """Remove cached results made from sidecars that have just changed.""" + for rel in file_list: + result_stem = os.path.splitext(rel)[0] + bug_id = result_stem.replace(os.sep, "--").replace("/", "--") + stale_paths = ( + os.path.join(output_dir, result_stem + ".json"), + os.path.join(work_dir, "bug_validation", f"{bug_id}.result.json"), + os.path.join(work_dir, "bug_validation", f"{bug_id}.md"), + ) + for stale_path in stale_paths: + try: + os.remove(stale_path) + logging.info("Removed stale verification artifact: %s", stale_path) + except FileNotFoundError: + pass + + if file_list: + try: + os.remove(os.path.join(work_dir, "bug_validation", "summary.json")) + logging.info("Removed stale bug-validation summary.") + except FileNotFoundError: + pass + + +def _clear_stale_sidecars_before_attempt(function_paths): + """Remove incomplete canonical and bare sidecars before an attempt. + + A valid bare spec and info file can otherwise come from different retry + attempts. A valid canonical half can likewise be read by streaming_reasoner + while the producer publishes its replacement half. This function runs + before both producers and their watcher start, so clearing all four paths + makes any subsequently ready pair belong to the new attempt. + """ + stale_paths = set() + for function_path in function_paths: + base = os.path.splitext(function_path)[0] + stale_paths.add(f"{function_path}.spec.json") + stale_paths.add(f"{function_path}.info.json") + stale_paths.add(f"{base}.spec.json") + stale_paths.add(f"{base}.info.json") + + for stale_path in sorted(stale_paths): + try: + os.remove(stale_path) + logging.info("Removed stale sidecar before generation attempt: %s", stale_path) + except FileNotFoundError: + pass + + def _run_spec_generation_batch( proj_dir, work_dir, @@ -39,6 +94,7 @@ def _run_spec_generation_batch( layer_idx, batch_rel_dir, batch_info, + normalization_candidates=None, ): # Run one batch end-to-end so the executor can refill slots as soon as a # batch finishes, instead of waiting for a whole chunk barrier. @@ -108,6 +164,14 @@ def _run_spec_generation_batch( return result.returncode except subprocess.CalledProcessError as exc: return exc.returncode + finally: + # Publish valid extension-dropped sidecars before this Future becomes + # done. The existing streaming_reasoner can then verify this batch while + # other spec-generation batches are still running. + normalize_spec_filenames( + [os.path.join(proj_dir, func_rel) for func_rel in function_files], + all_function_files=normalization_candidates, + ) def run_spec_generation_and_verification( @@ -179,6 +243,9 @@ def run_spec_generation_and_verification( layer_files.append(rel) layer_processed = set() + layer_function_paths = [ + os.path.join(input_dir, rel) for rel in layer_files + ] for attempt in range(1, OPENCODE_MAX_RETRIES + 1): # Find batches with unspecced functions @@ -214,6 +281,34 @@ def run_spec_generation_and_verification( layer_processed.update(newly_processed) break + # Any function entering spec generation has invalid or missing + # sidecars, so verification artifacts made from its previous + # sidecars are stale. Invalidate them before producers start so + # the running watcher cannot resume an old verdict when either + # canonical or normalized sidecars become ready. + pending_rel_paths = [] + pending_rel_seen = set() + pending_abs_paths = set() + for batch_info in pending_batches: + for func_rel in batch_info.get("functions", []): + func_path = os.path.join(proj_dir, func_rel) + if is_file_ready(func_path): + continue + rel = os.path.relpath(func_path, input_dir) + if rel in pending_rel_seen: + continue + pending_rel_seen.add(rel) + pending_rel_paths.append(rel) + pending_abs_paths.add(func_path) + + _clear_stale_sidecars_before_attempt(pending_abs_paths) + _invalidate_verification_artifacts( + pending_rel_paths, + output_dir, + work_dir, + ) + layer_processed.difference_update(pending_abs_paths) + # Submit all pending spec batches through a bounded executor so # finished slots can immediately pick up the next batch. spec_futures = [] @@ -238,6 +333,7 @@ def run_spec_generation_and_verification( layer_idx, batch_rel_dir, batch_info, + layer_function_paths, ) ) @@ -251,7 +347,7 @@ def run_spec_generation_and_verification( input_dir, output_dir, file_list=layer_files, proj_dir=proj_dir, work_dir=work_dir, spec_procs=spec_futures, - already_processed=all_processed | layer_processed, + already_processed=(all_processed | layer_processed) - pending_abs_paths, resume=resume, bug_validator_path=bug_validator_path, all_bugs=all_bugs, diff --git a/src/verification.py b/src/verification.py index 99fe0066..4b1dcca0 100644 --- a/src/verification.py +++ b/src/verification.py @@ -123,6 +123,7 @@ def streaming_reasoner( reasoning_futures = {} validation_futures = {} submitted = set() + producers_done_observed = False while True: # Scan for new ready files @@ -260,6 +261,14 @@ def streaming_reasoner( # Detect if spec generation subprocesses exited before all files are ready _all_procs = spec_procs if spec_procs else None if _all_procs is not None and all(_spec_task_done(p) for p in _all_procs): + if not producers_done_observed: + # A producer can publish its normalized sidecars and + # become done between the scan at the top of this loop + # and this check. Rescan once before deciding that any + # remaining files are genuinely unready. + producers_done_observed = True + continue + unready = (expected_files or set()) - processed if unready and not reasoning_futures and not validation_futures: exit_codes = [_spec_task_exit_code(p) for p in _all_procs]