Fix #6957: Replace OS-specific bash commands with cross-platform inline Python

benchmarks/nel/project.yml

@@ -155,8 +155,8 @@ commands:
   - name: clean
     help: "Remove intermediate files for specified dataset and language (excluding Wiki resources and database)."
     script:
-      - "rm -rf training/${vars.dataset}"
-      - "rm -rf corpora/${vars.dataset}"
-      - "rm -rf assets/${vars.dataset}"
-      - "rm -rf evaluation/${vars.dataset}"
-      - "rm -rf wikid/output/${vars.language}"
+      - python -c "import shutil, os; shutil.rmtree('training/${vars.dataset}', ignore_errors=True); os.makedirs('training/${vars.dataset}', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpora/${vars.dataset}', ignore_errors=True); os.makedirs('corpora/${vars.dataset}', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('assets/${vars.dataset}', ignore_errors=True); os.makedirs('assets/${vars.dataset}', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('evaluation/${vars.dataset}', ignore_errors=True); os.makedirs('evaluation/${vars.dataset}', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('wikid/output/${vars.language}', ignore_errors=True); os.makedirs('wikid/output/${vars.language}', exist_ok=True)"

benchmarks/ner_conll03/project.yml

@@ -77,6 +77,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

benchmarks/ner_embeddings/project.yml

@@ -283,7 +283,7 @@ commands:
   - name: "evaluate"
     help: "Evaluate NER model."
     script:
-      - mkdir -p ${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
+      - python -c "import os; os.makedirs('${vars.metrics_dir}/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
@@ -315,7 +315,7 @@ commands:
         --output ${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
         --gpu-id ${vars.gpu_id}
         --code scripts/custom_components.py
-      - mkdir -p ${vars.metrics_dir}-test-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
+      - python -c "import os; os.makedirs('${vars.metrics_dir}-test-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
@@ -335,23 +335,23 @@ commands:
   - name: "evaluate-unseen-only"
     help: "Evaluate NER model on the dev and tests sets only considering entities that did not appear in the training set."
     script:
-      - mkdir -p ${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
+      - python -c "import os; os.makedirs('${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
         corpus/${vars.dataset}-dev-unseen.spacy
         --output ${vars.metrics_dir}-dev-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
         --gpu-id ${vars.gpu_id}
         --code scripts/custom_components.py
-      - mkdir -p ${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
+      - python -c "import os; os.makedirs('${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
         corpus/${vars.dataset}-test-unseen.spacy
         --output ${vars.metrics_dir}-test-unseen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/scores.json
         --gpu-id ${vars.gpu_id}
         --code scripts/custom_components.py
-      - mkdir -p ${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/
+      - python -c "import os; os.makedirs('${vars.metrics_dir}-dev-seen/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/', exist_ok=True)"
     deps:
       - training/${vars.dataset}/${vars.vectors}/${vars.ner_config}/${vars.seed}/model-best
       - corpus/${vars.dataset}-dev-unseen.spacy

benchmarks/parsing_penn_treebank/project.yml

@@ -35,7 +35,7 @@ commands:
     # Make sure we specify the branch in the command string, so that the
     # caching works correctly.
     script:
-      - "mkdir -p assets/PTB_SD_3_3_0_pos_fix"
+      - python -c "import os; os.makedirs('assets/PTB_SD_3_3_0_pos_fix', exist_ok=True)"
       - "python scripts/fix_conll_format.py assets/PTB_SD_3_3_0/train.gold.conll assets/PTB_SD_3_3_0_pos_fix/train.conll"
       - "python scripts/fix_conll_format.py assets/PTB_SD_3_3_0/dev.gold.conll assets/PTB_SD_3_3_0_pos_fix/dev.conll"
       - "python scripts/fix_conll_format.py assets/PTB_SD_3_3_0/test.gold.conll assets/PTB_SD_3_3_0_pos_fix/test.conll"
@@ -83,6 +83,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

benchmarks/speed/project.yml

@@ -74,4 +74,4 @@ commands:
   - name: clean
     help: "Remove output file(s)"
     script:
-      - "rm -rf results/*"
+      - python -c "import shutil, os; shutil.rmtree('results', ignore_errors=True); os.makedirs('results', exist_ok=True)"

benchmarks/ud_benchmark/project.yml

@@ -134,6 +134,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

experimental/ner_spancat/project.yml

@@ -68,6 +68,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

experimental/ner_spancat_compare/project.yml

@@ -185,7 +185,7 @@ commands:
   - name: "evaluate-ner"
     help: "Evaluate all NER models"
     script:
-      - "mkdir -p metrics/ner"
+      - python -c "import os; os.makedirs('metrics/ner', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/ner/${vars.dna}/model-best
@@ -250,7 +250,7 @@ commands:
   - name: "evaluate-spancat"
     help: "Evaluate SpanCat model"
     script:
-      - "mkdir -p metrics/spancat"
+      - python -c "import os; os.makedirs('metrics/spancat', exist_ok=True)"
       - >-
         python -m spacy evaluate
         training/spancat/model-best

experimental/ner_wikiner_speedster/project.yml

@@ -66,6 +66,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

integrations/ray/project.yml

@@ -38,7 +38,7 @@ commands:
     # Make sure we specify the branch in the command string, so that the
     # caching works correctly.
     script:
-      - "mkdir -p corpus/${vars.treebank}"
+      - python -c "import os; os.makedirs('corpus/${vars.treebank}', exist_ok=True)"
       - "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"
       - "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"
       - "python -m spacy convert assets/${vars.treebank}/${vars.test_name}.conllu corpus/${vars.treebank}/ --converter conllu --n-sents 10"

integrations/wandb/project.yml

@@ -64,5 +64,5 @@ commands:
   - name: "clean"
     help: "Remove intermediate files."
     script:
-      - "rm -rf training/*"
-      - "rm -rf wandb/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('wandb', ignore_errors=True); os.makedirs('wandb', exist_ok=True)"

pipelines/edit_tree_lemmatizer/project.yml

@@ -28,7 +28,7 @@ commands:
   - name: "preprocess"
     help: "Convert data to spaCy format"
     script:
-      - "mkdir -p corpus/${vars.treebank}"
+      - python -c "import os; os.makedirs('corpus/${vars.treebank}', exist_ok=True)"
       - "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
       - "mv corpus/${vars.treebank}/${vars.train_name}.spacy corpus/${vars.treebank}/train.spacy"
       - "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --n-sents 10"

pipelines/floret_fi_core_demo/project.yml

@@ -126,7 +126,7 @@ commands:
   - name: "convert"
     help: "Convert the data to spaCy's format"
     script:
-      - "mkdir -p corpus/${vars.treebank}"
+      - python -c "import os; os.makedirs('corpus/${vars.treebank}', exist_ok=True)"
       - "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
       - "mv corpus/${vars.treebank}/${vars.train_name}.spacy corpus/${vars.treebank}/train.spacy"
       - "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
@@ -190,13 +190,13 @@ commands:
   - name: "evaluate"
     help: "Evaluate the models and export metrics"
     script:
-      - "mkdir -p metrics/${vars.treebank}-novectors"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}-novectors', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}-novectors/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}-novectors/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/metrics.json --gpu-id ${vars.gpu_id}"
     deps:
       - "corpus/${vars.treebank}/test.spacy"
@@ -213,7 +213,7 @@ commands:
   - name: "convert-ner"
     help: "Convert the data to spaCy's format"
     script:
-      - "mkdir -p corpus/${vars.corpus_ner}"
+      - python -c "import os; os.makedirs('corpus/${vars.corpus_ner}', exist_ok=True)"
       - "python -m spacy convert assets/${vars.corpus_ner}/data/conll/train.tsv corpus/${vars.corpus_ner}/ -c ner --n-sents 10"
       - "python -m spacy convert assets/${vars.corpus_ner}/data/conll/dev.tsv corpus/${vars.corpus_ner}/ -c ner --n-sents 10"
       - "python -m spacy convert assets/${vars.corpus_ner}/data/conll/test.tsv corpus/${vars.corpus_ner}/ -c ner --n-sents 10"
@@ -274,13 +274,13 @@ commands:
   - name: "evaluate-ner"
     help: "Evaluate the models and export metrics"
     script:
-      - "mkdir -p metrics/${vars.corpus_ner}-novectors"
+      - python -c "import os; os.makedirs('metrics/${vars.corpus_ner}-novectors', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.corpus_ner}-novectors/model-best corpus/${vars.corpus_ner}/test.spacy --output metrics/${vars.corpus_ner}-novectors/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned"
+      - python -c "import os; os.makedirs('metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/model-best corpus/${vars.corpus_ner}/test.spacy --output metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard"
+      - python -c "import os; os.makedirs('metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/model-best corpus/${vars.corpus_ner}/test.spacy --output metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret"
+      - python -c "import os; os.makedirs('metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/model-best corpus/${vars.corpus_ner}/test.spacy --output metrics/${vars.corpus_ner}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/metrics.json --gpu-id ${vars.gpu_id}"
     deps:
       - "corpus/${vars.corpus_ner}/test.spacy"

pipelines/floret_ko_ud_demo/project.yml

@@ -113,7 +113,7 @@ commands:
   - name: "convert"
     help: "Convert the data to spaCy's format"
     script:
-      - "mkdir -p corpus/${vars.treebank}"
+      - python -c "import os; os.makedirs('corpus/${vars.treebank}', exist_ok=True)"
       - "python -m spacy convert assets/${vars.treebank}/${vars.train_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
       - "mv corpus/${vars.treebank}/${vars.train_name}.spacy corpus/${vars.treebank}/train.spacy"
       - "python -m spacy convert assets/${vars.treebank}/${vars.dev_name}.conllu corpus/${vars.treebank}/ --n-sents 10"
@@ -177,13 +177,13 @@ commands:
   - name: "evaluate"
     help: "Evaluate the models and export metrics"
     script:
-      - "mkdir -p metrics/${vars.treebank}-novectors"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}-novectors', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}-novectors/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}-novectors/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard-unpruned/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-standard/metrics.json --gpu-id ${vars.gpu_id}"
-      - "mkdir -p metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret"
+      - python -c "import os; os.makedirs('metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret', exist_ok=True)"
       - "python -m spacy evaluate training/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/model-best corpus/${vars.treebank}/test.spacy --output metrics/${vars.treebank}.${vars.max_texts}.n${vars.minn}-${vars.maxn}-floret/metrics.json --gpu-id ${vars.gpu_id}"
     deps:
       - "corpus/${vars.treebank}/test.spacy"

pipelines/ner_wikiner/project.yml

@@ -63,6 +63,6 @@ commands:
   - name: clean
     help: "Remove intermediate files"
     script:
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
-      - "rm -rf corpus/*"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"

pipelines/spancat_demo/project.yml

@@ -131,6 +131,6 @@ commands:
   - name: clean
     help: "Remove intermediary directories"
     script:
-      - "rm -rf corpus/*"
-      - "rm -rf training/*"
-      - "rm -rf metrics/*"
+      - python -c "import shutil, os; shutil.rmtree('corpus', ignore_errors=True); os.makedirs('corpus', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('training', ignore_errors=True); os.makedirs('training', exist_ok=True)"
+      - python -c "import shutil, os; shutil.rmtree('metrics', ignore_errors=True); os.makedirs('metrics', exist_ok=True)"