From 3a0158cafa8db1e93c6084c8bf64460b64a58b45 Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Fri, 12 May 2023 10:46:59 -0500 Subject: [PATCH 01/10] Replace hyphen dash with ascii dash. --- src/gdcdictionary/schemas/aligned_reads.yaml | 6 +++--- src/gdcdictionary/schemas/diagnosis.yaml | 6 +++--- src/gdcdictionary/schemas/molecular_test.yaml | 20 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/gdcdictionary/schemas/aligned_reads.yaml b/src/gdcdictionary/schemas/aligned_reads.yaml index f05be12d0..c934d72c3 100644 --- a/src/gdcdictionary/schemas/aligned_reads.yaml +++ b/src/gdcdictionary/schemas/aligned_reads.yaml @@ -226,9 +226,9 @@ properties: $ref: - "_terms.yaml#/wgs_coverage/common" enum: - - 0x–10x - - 10x–25x - - 25x–150x + - 0x-10x + - 10x-25x + - 25x-150x - 150x+ - Unknown - Not Applicable diff --git a/src/gdcdictionary/schemas/diagnosis.yaml b/src/gdcdictionary/schemas/diagnosis.yaml index f128b10d1..f60164a6e 100644 --- a/src/gdcdictionary/schemas/diagnosis.yaml +++ b/src/gdcdictionary/schemas/diagnosis.yaml @@ -2092,9 +2092,9 @@ properties: The length of time the first presenting symptom occurred that related to the diagnosis of the patient with cancer. enum: - - 0–30 Days - - 31–90 Days - - 91–180 Days + - 0-30 Days + - 31-90 Days + - 91-180 Days - '>=181 Days' first_symptom_prior_to_diagnosis: diff --git a/src/gdcdictionary/schemas/molecular_test.yaml b/src/gdcdictionary/schemas/molecular_test.yaml index 179c1d07e..e4daef9e9 100644 --- a/src/gdcdictionary/schemas/molecular_test.yaml +++ b/src/gdcdictionary/schemas/molecular_test.yaml @@ -7251,17 +7251,17 @@ properties: - 51-75 - 76-100 - <10% - - 10–19% - - 20–29% - - 30–39% - - 40–49% - - 50–59% - - 60–69% - - 70–79% - - 80–89% - - 90–99% + - 10-19% + - 20-29% + - 30-39% + - 40-49% + - 50-59% + - 60-69% + - 70-79% + - 80-89% + - 90-99% - <1% - - 1–49% + - 1-49% - '>=50%' - Not Reported From ca64a9645536fa29dc56bdaa26ee58bbccbbf759 Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Fri, 12 May 2023 20:21:55 -0500 Subject: [PATCH 02/10] DEV-1849: Add yamllint, and yamlfmt manual-stage only. yamllint is now on always. yamlfmt is only enabled during manual stage. E.g. $ pre-commit run --hook-stage manual yamlfmt --files src/gdcdictionary/schemas/aligned_reads.yaml The idea is to convert a few files at the time until they're all well formatted. --- .pre-commit-config.yaml | 48 ++++++++++++++++++++++++++++++----------- .yamllint | 11 ++++++++++ docker-compose-ci.yaml | 7 +++--- 3 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 .yamllint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d7590cda..67fae73c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,47 @@ +--- repos: -- repo: git@github.com:Yelp/detect-secrets + - repo: git@github.com:Yelp/detect-secrets rev: v0.13.0 hooks: - - id: detect-secrets - args: ['--baseline', '.secrets.baseline'] + - id: detect-secrets + args: [--baseline, .secrets.baseline] -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 hooks: - - id: check-json - - id: check-toml - - id: check-yaml + - id: check-json + - id: check-toml + - id: check-yaml exclude: .gitlab-ci.yml - - id: end-of-file-fixer - - id: fix-encoding-pragma + - id: end-of-file-fixer + - id: fix-encoding-pragma args: [--remove] - - id: no-commit-to-branch + - id: no-commit-to-branch args: [--branch, develop, --branch, master, --pattern, release/.*] - - id: pretty-format-json + - id: pretty-format-json args: [--autofix, --no-sort-keys] - - id: trailing-whitespace + - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + + - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt + rev: 0.2.1 + hooks: + - id: yamlfmt + args: [--mapping, '2', --sequence, '4', --offset, '2', --width, '130'] + # TODO: Remove stages so it always run. Do it once all yamls are properly formatted. + stages: [manual] + + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.31.0 + hooks: + - id: yamllint + name: Lint YAML files + args: [--strict] + files: ^src/.*\.(yaml|yml)$ + exclude: | + (?x)^( + sandbox/.*\.(yaml|yml)| + \.yamllint| + \.pre-commit-config\.yaml| + docker-compose-ci\.yaml + )$ diff --git a/.yamllint b/.yamllint new file mode 100644 index 000000000..8b8a0d2be --- /dev/null +++ b/.yamllint @@ -0,0 +1,11 @@ +--- +# SEE https://yamllint.readthedocs.io/en/stable/configuration.html#default-configuration + +extends: default + +rules: + document-start: disable + line-length: + max: 130 + indentation: + indent-sequences: consistent diff --git a/docker-compose-ci.yaml b/docker-compose-ci.yaml index 4d0749683..628e234b7 100644 --- a/docker-compose-ci.yaml +++ b/docker-compose-ci.yaml @@ -1,4 +1,5 @@ -version: "3.3" +--- +version: '3.3' services: app: image: quay.io/ncigdc/jenkins-agent:multipython @@ -7,6 +8,6 @@ services: HTTPS_PROXY: http://cloud-proxy:3128 HTTP_PROXY: http://cloud-proxy:3128 volumes: - - .:/home/jenkins - - $SSH_AUTH_SOCK:$SSH_AUTH_SOCK + - .:/home/jenkins + - $SSH_AUTH_SOCK:$SSH_AUTH_SOCK command: bash -c "tox --recreate" From b2b0ecbae971a2c01b5a3ffa1a735dc0477b0d8c Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Fri, 12 May 2023 22:51:28 -0500 Subject: [PATCH 03/10] DEV 1849 Add local hook 'replacer'. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An script that sed all schema files and replaces unicode en-dash “–” (U+2013) with ascii dash. --- .pre-commit-config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67fae73c7..3d0f8d456 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,23 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + - repo: local + hooks: + - id: replacer + # More chars can be added + name: Replace unicode dashes with ascii dashes. + language: system + pass_filenames: false + entry: | + sh -c " + # echo -n '–' | hexdump -C # > e2 80 93 + LC_ALL=C + for f in $(ls src/gdcdictionary/schemas/*.yaml); do + # echo 'Converting ' $f + sed -i '' 's/\xe2\x80\x93/-/g' $f + done + " + - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt rev: 0.2.1 hooks: From f112757ed55e967b5661b7d7d5d4ba7bc7db0105 Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Tue, 16 May 2023 10:56:44 -0500 Subject: [PATCH 04/10] Unless 'env' is in 'allowlist_externals', it will not run until completion. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6318a9d62..adffd2b0f 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ skip_missing_interpreters = true isolated_build = True [testenv] +allowlist_externals = env deps = -rdev-requirements.txt commands = From e1152b460fd26ce900accb0fc8ddf7031f378aaf Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Tue, 16 May 2023 11:22:45 -0500 Subject: [PATCH 05/10] Update import. 'src' is not a package. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index a9685ab93..ff0b8126d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import os import pytest -from src.gdcdictionary import ROOT_DIR, GDCDictionary +from gdcdictionary import ROOT_DIR, GDCDictionary from tests.utils import load_yaml From 0ca8a89e306b58982b76ee322d3c58cea0405275 Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Tue, 16 May 2023 12:24:19 -0500 Subject: [PATCH 06/10] Fix commands setting. According to [tox docs], the exit code of any command preceeded by a dash is ignored. [tox docs]: https://tox.wiki/en/latest/config.html#commands --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index adffd2b0f..61ab1f52f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,5 +8,5 @@ allowlist_externals = env deps = -rdev-requirements.txt commands = - - env - - pytest -lvv --cov=gdcdictionary --cov-report xml --cov-report html --junit-xml test-reports/results.xml {posargs} + env + pytest -lvv --cov=gdcdictionary --cov-report xml --cov-report html --junit-xml test-reports/results.xml {posargs} From 467c39e07985023114401e63c50c32a1a76e15cc Mon Sep 17 00:00:00 2001 From: Henddher Pedroza Date: Tue, 16 May 2023 14:35:41 -0500 Subject: [PATCH 07/10] Update exon ref. --- src/gdcdictionary/schemas/tag.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gdcdictionary/schemas/tag.yaml b/src/gdcdictionary/schemas/tag.yaml index 40581cab4..4f4c7e6b6 100644 --- a/src/gdcdictionary/schemas/tag.yaml +++ b/src/gdcdictionary/schemas/tag.yaml @@ -119,4 +119,4 @@ properties: enumDef: exon: $ref: - - "_terms_enum.yaml#/exon/common" + - "_terms_enum.yaml#/exon/tag/name" From 0832804b897964e55159a47b2aadb3e1d3b49456 Mon Sep 17 00:00:00 2001 From: wwysoc2 Date: Thu, 11 May 2023 11:56:14 -0500 Subject: [PATCH 08/10] TERMS - typo fixes --- src/gdcdictionary/schemas/_terms.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gdcdictionary/schemas/_terms.yaml b/src/gdcdictionary/schemas/_terms.yaml index a3ae52785..e0a9209ee 100644 --- a/src/gdcdictionary/schemas/_terms.yaml +++ b/src/gdcdictionary/schemas/_terms.yaml @@ -2660,8 +2660,8 @@ height: hepatitis_sustained_virological_response: common: description: >- - The yes/no/unknown indicator used to describe whether the patient received treatment - for a risk factor the patient had at the time of or prior to their diagnosis. + An indication as to whether sustained virological response was obtained in responses + to viral hepatitis treatment. termDef: term: >- Viral Hepatitis Sustained Virologic Response Post Treatment Indicator @@ -6745,7 +6745,7 @@ viral_hepatitis_serologies: common: description: >- Text term that describes the kind of serological laboratory test used to determine - the patient's hepatitus status. + the patient's hepatitis status. termDef: term: >- Viral Hepatitis Laboratory Procedure Serology Test Method Type @@ -6825,7 +6825,7 @@ weiss_assessment_score: well_number: common: description: >- - Numeric value that represents the the well location within a plate for the analyte + Numeric value that represents the well location within a plate for the analyte or aliquot from the sample. termDef: term: >- @@ -7280,7 +7280,7 @@ spindle_cell_percent: common: description: >- The percent of uveal melanoma arising from the choroid, ciliary body, or the - iris and characterized by the presence of of spindle-shaped melanocytes. + iris and characterized by the presence of spindle-shaped melanocytes. termDef: term: >- Cell Morphology Percent Uveal Spindle Cell Melanoma Integer From ad821ed9b590184b85aab6e7bad7b5162905800e Mon Sep 17 00:00:00 2001 From: wwysoc2 Date: Thu, 11 May 2023 13:23:20 -0500 Subject: [PATCH 09/10] Added WGS definition --- src/gdcdictionary/schemas/_terms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gdcdictionary/schemas/_terms.yaml b/src/gdcdictionary/schemas/_terms.yaml index e0a9209ee..2b92ca338 100644 --- a/src/gdcdictionary/schemas/_terms.yaml +++ b/src/gdcdictionary/schemas/_terms.yaml @@ -7362,7 +7362,7 @@ country_of_birth: wgs_coverage: common: description: >- - Range of coverage values for WGS aligned reads. + Range of coverage values for WGS aligned reads. Each range is non-inclusive to the lower bound. termDef: term: source: From 70dc786a854fc16d8a98e79afcd855c2daf3f926 Mon Sep 17 00:00:00 2001 From: Catherine Date: Tue, 16 May 2023 12:05:42 -0500 Subject: [PATCH 10/10] update ensat_pathologic_state enumDef links --- src/gdcdictionary/schemas/diagnosis.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gdcdictionary/schemas/diagnosis.yaml b/src/gdcdictionary/schemas/diagnosis.yaml index f60164a6e..d11221663 100644 --- a/src/gdcdictionary/schemas/diagnosis.yaml +++ b/src/gdcdictionary/schemas/diagnosis.yaml @@ -1839,10 +1839,10 @@ properties: - "_terms_enum.yaml#/stage_i/diagnosis/ensat_pathologic_stage" Stage II: $ref: - - "_terms_enum.yaml#/stage_iI/diagnosis/ensat_pathologic_stage" + - "_terms_enum.yaml#/stage_ii/diagnosis/ensat_pathologic_stage" Stage III: $ref: - - "_terms_enum.yaml#/stage_iII/diagnosis/ensat_pathologic_stage" + - "_terms_enum.yaml#/stage_iii/diagnosis/ensat_pathologic_stage" Stage IV: $ref: - "_terms_enum.yaml#/stage_iv/diagnosis/ensat_pathologic_stage"