From dbd978157d09b4224dae02d25cd71049efbf789c Mon Sep 17 00:00:00 2001 From: Dimos Tzimotoudis Date: Mon, 13 Jul 2026 01:02:39 +0000 Subject: [PATCH 1/2] Discourage checkpointing large training caches --- src/agentomics/agents/prompt_builder.py | 2 ++ src/agentomics/agents/steps/model_training.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/agentomics/agents/prompt_builder.py b/src/agentomics/agents/prompt_builder.py index c0dc8b74..31ca3905 100644 --- a/src/agentomics/agents/prompt_builder.py +++ b/src/agentomics/agents/prompt_builder.py @@ -76,6 +76,8 @@ def build_iteration_base_prompt(config: Config, iteration: int) -> str: Workspace rules: - Your current writable directory is: {config.current_step_dir} - Create and modify files only inside your current writable directory. + - Files left in your current writable directory are checkpointed after the step. Do not leave regenerable large caches, downloaded package/model caches, full-dataset tensors/embeddings, or large diagnostic dumps there. + - If you need temporary caches while working, keep them clearly named inside your current writable directory and delete them before final_result. - Previous step and iteration folders are read-only. - If you want to reuse earlier files, copy them into your current writable directory before modifying them. - Don't create or modify any folders or files starting with 'iteration_'. diff --git a/src/agentomics/agents/steps/model_training.py b/src/agentomics/agents/steps/model_training.py index f9d08f12..0084f053 100644 --- a/src/agentomics/agents/steps/model_training.py +++ b/src/agentomics/agents/steps/model_training.py @@ -137,7 +137,9 @@ def step_prompt(self) -> str: Your next task: implement training code and train your model. Training guidelines: - Train until validation performance stops improving, and output the best checkpoint. - - Save all artifacts needed for inference (model file, tokenizers, etc...). + - Save only artifacts needed for inference (model file, tokenizers, small preprocessing metadata, small copied source files needed by inference, etc...). + - Do not save regenerable training caches, full train/validation feature tensors, embedding arrays, PHACT tensor arrays, downloaded package caches, or prediction-diagnostic dumps in training_artifacts or leave them elsewhere in the step directory. + - If large intermediate arrays are needed during training, keep them in a temporary cache directory under the current step directory, outside training_artifacts, and delete that cache before the script exits. Do not leave directories/files such as _training_cache, _embedding_cache_tmp, *embedding_cache*, train_*.npy, or val_*.npy in final step outputs. - If you failed to implement your intended model, when you call the final_result tool, put into unresolved issues what went wrong. {"- If your model can be accelerated by GPU, implement the code to use GPU." if check_gpu_availability() else ""} {reporting_requirement} @@ -146,7 +148,7 @@ def step_prompt(self) -> str: The train script should take the following parameters --train-data (a path to the training split folder. This folder contains input/ with the data files and labels.csv with labels keyed by id.) --validation-data (a path to the validation split folder. This folder contains input/ with the data files and labels.csv with labels keyed by id. For example for the purposes of early-stopping. If the training script doesn't need validation data, still include the argument for compatibility and don't use it.) - --artifacts-dir (path to a directory that will be populated by the training script with artifacts needed to use the trained model for predictions (e.g. produced model weights, produced tokenizers, ...). This directory should not contain any other external sources like imported scripts, conda packages, foundation models, etc..) + --artifacts-dir (path to a directory that will be populated by the training script with artifacts needed to use the trained model for predictions (e.g. produced model weights, produced tokenizers, ...). This directory should contain only final inference artifacts and must not contain training caches, full train/validation tensors, downloaded package caches, or other regenerable intermediate files. This directory should not contain any other external sources like imported scripts, conda packages, foundation models, etc..) The script must not accept any other parameters. Data paths: From 7cd69a4128d847638c1f4b5c25f88c143d2a7c51 Mon Sep 17 00:00:00 2001 From: Dimos Tzimotoudis Date: Fri, 31 Jul 2026 12:53:45 +0000 Subject: [PATCH 2/2] generalize cache cleanup guidance --- src/agentomics/agents/steps/model_training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agentomics/agents/steps/model_training.py b/src/agentomics/agents/steps/model_training.py index 0084f053..3ddb531d 100644 --- a/src/agentomics/agents/steps/model_training.py +++ b/src/agentomics/agents/steps/model_training.py @@ -138,8 +138,8 @@ def step_prompt(self) -> str: Training guidelines: - Train until validation performance stops improving, and output the best checkpoint. - Save only artifacts needed for inference (model file, tokenizers, small preprocessing metadata, small copied source files needed by inference, etc...). - - Do not save regenerable training caches, full train/validation feature tensors, embedding arrays, PHACT tensor arrays, downloaded package caches, or prediction-diagnostic dumps in training_artifacts or leave them elsewhere in the step directory. - - If large intermediate arrays are needed during training, keep them in a temporary cache directory under the current step directory, outside training_artifacts, and delete that cache before the script exits. Do not leave directories/files such as _training_cache, _embedding_cache_tmp, *embedding_cache*, train_*.npy, or val_*.npy in final step outputs. + - Do not save regenerable caches, intermediate data, downloaded resources, or diagnostic outputs in training_artifacts or leave them elsewhere in the step directory. + - If large intermediate files are needed during training, keep them in a temporary cache directory under the current step directory, outside training_artifacts, and delete that cache before the script exits. - If you failed to implement your intended model, when you call the final_result tool, put into unresolved issues what went wrong. {"- If your model can be accelerated by GPU, implement the code to use GPU." if check_gpu_availability() else ""} {reporting_requirement}