From 64c61b9bfcef237801bfe618e43c3108f51ed726 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Wed, 25 Mar 2026 16:39:35 +0100 Subject: [PATCH 01/28] rmv ds_store --- .DS_Store | Bin 10244 -> 0 bytes .gitignore | 4 +++- docs/.DS_Store | Bin 8196 -> 0 bytes res/.DS_Store | Bin 6148 -> 0 bytes src/.DS_Store | Bin 6148 -> 0 bytes src/rarelink/.DS_Store | Bin 6148 -> 0 bytes src/rarelink/tofhir/.DS_Store | Bin 6148 -> 0 bytes tests/.DS_Store | Bin 6148 -> 0 bytes 8 files changed, 3 insertions(+), 1 deletion(-) delete mode 100755 .DS_Store delete mode 100755 docs/.DS_Store delete mode 100755 res/.DS_Store delete mode 100755 src/.DS_Store delete mode 100755 src/rarelink/.DS_Store delete mode 100755 src/rarelink/tofhir/.DS_Store delete mode 100755 tests/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100755 index a422ca80f231f03a39a13ad21f46458baef38753..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10244 zcmeHMUu+ab7@u!@(A_PyTWm|Y4RE1COF87O*p{{!uI;r_5J9e`rGV$%?Y7-;y)CzU zSD+y_5?}bEs8R4i{QoG?pvI7(kQgPtX{y4DQH(zLCqaYyqQP%==i0rt5PTqFvXkt7 zJKvxEW`6UVnVUUA2()FCAR#_N2#*()(mcG)QmCC?GYZEvRVV@DNpdh4N$A5PGdx{| zi-3!Oi-3!Oi-3#3oq_=F*|hLQwsh|<0xkkB0+R^P^}&Z1l_4g_ZD}nXc(aZGC_SjQ z3O-Bp0i}=05R>D!L<<(MLs8nHz>XMThoj!d{6b8Q+tLmvzz!e4G6OrIfR!EJ@3Y(q zgly^FT?AYNW+Fh(?jCY486+Apo#*#zZM)#K4eQ}>JgLV~$3HDzJAUNk$F2Ex<)-$L zp{Z{JE5!2{N(ys%$q>nqxHCH;!k;reoNcH$uzAC|POA;m-ZQLEwdBTL&i1gi%0m>Q zlLRs1qs9p9>-+JMq-L0bz?Z-kc;?J4EEEch3U^0_^wEeJGc!h8EWMM~x^+Dn&ppTD z+ckAGDwVbCDKn;})U*|#DJr_`>eW(;KH8>dQi@^4Mr?y1@Iq9o92;v4)cbwGK+{3L zZ>(ue!0%gI*KqKlz?WC8ZrK*ylN?Uzhbe%#%(3N=6x;R5`w@Ja?2lq3l5uPwb)Hl@ z;r4t`v&?C80{wT#H)qzIV1vy7azZ?t-3hK|!iTS(%A1X(Q%OBFh&tMy0+J?b9^_|d zKwfwoAr}yG@7P#yZBw1!S63IHmEMV@ULF*QA9Y**6;%BWOgARoID zdTMK0rS{&0p3DrVjHpzah$R!5WXx1slgWs7KsEYIb)QL}`VBp8wp)%-Ny-@NHWmG} ztQtmdqff)vQMWP*fTMtLdS3aWrOQ_a>YLYxyEZ>OQG8d4R61W2`;pWsQyb9KbkC@! zm_s{aMnX*~T57O2s~K86sdlGg!)ht?2*N~h#e&L(URl18YhNa=?agTBeyNJ*tsIM& z$l@*q8@E|kXck3!m&f;D_Npbaypvkm(< zVmTuB@Y3fzj+2(kYPbyHu-mHZT!t%f z4St6|;5sL8B3Hzfapl|su992H`MG+onQO7z&gB+4IzJqR3u7fWlZ-4|vbnvp%@=Ry z17g=5F<&mvvuDWNW#x;jYF5=Xw1g&^WK%E4HYbNI;^xuaBCa*c(-=2lPbG0tZkkP> zQ$5IdbstY@D;3(_K5;3JjF;xv)n#Hek6f4L+SOXGpU2rHaaL7a?F|Yzhol0#TI;PB zcnR#vTCrKc*(7=F>iyz+flhFVSaolkH_YRVkobR3)VIl5a-Liy6G+q_$WP=dxeh$i zvjTju90^M4xe6Mf5t^VG)LjtKf0E3`G8VoQY3y;A#()AEL zfnvh?iiXaOv+K;-D?i#BCnKqI+ni~2 zZknXzDow)TW^9=TZkmbwEyhyiZ!wlCZ;N>$u*Z6Q~fE zYhGL5^3k@=Atv3(@owbhB)@*2Z*}Iq=j3EwN4yiK**)^|!*QD5EiM8s0xkkB0=J4l zf$iv{-~T&q{Qv*0zRvY?5pWUsA0hyra3tJ@4bPpkruTtgf~?T7pTHD-S3Sw0eNt-G?Ct z889F#-QR66twJoFbIKK__UvXiY}rYr49hVP6UzQ^e!ZOExYiwS?g?L+6K(m7UBf6$ zq1`Up?Lr+25;3zR?! z>yBN`7VW%C6^dpaA&>4AZPTeIotkZWR5#5U!r@RjGcvZYa4C8rqQ|1sixGWcIv$Pa zm!~c+E`~#=#wV}LWJ*@qc3u#((TP0R!*TX|Ctt#nkCo}=ECa11D!wLZIi3Bz+=>vL zzrJpk_$Ki6Bf7scN}K>VN6$`QKfYyh45z1xTc8XTzLE3b{U?*umYXympy@atZE@nP z%&sQBEj_1Oq64;h%=W@cGsNCE+xF?3r$O92WjtsLI9Hy;Vc_Z}a**0Z+lSw+d|77U z(1&T-+zBJuIzW$29BO}3KkH_U-2ZQM{{Fu|gQ;GW2b2d6r3bV?DxFHAY1`q}ZzZ{F z>-apuha`e)FiBM~qt|hi^g51m=O2bx))7i{5?nTz#2QQo{vklc{{wtUYa3Uk*a@1B YtDr4TjG?_^y7+F^3oFIcqZxb^G)pQmsa7L~C7t(T=(C fc6=R0S=W5c=e=-F3_9~cC+cUwb&*MdzgFNA6fhN( diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100755 index 4fc8327731a82309d271794bfffbe229251841ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK&x_MQ6n?X7U|zGgCL&{=k#SG7sK-lX>sW(uZ?^>dkV4;^g%=~TDw@nqBulVB(cLO;Nir;o$L z=aUv6Cw?kwT^X<})9O^OOs6}$&6?e)H)l0_+Ss{uqh{Z1?#^bGxqbEeod=!6C{Flm zsUO4bM%V>~b9jRtMJGCFxIG@rjp)VC*>NQ1?Y7#e%x{<8|8QS@xtovrYoddGZq}E- zux!5Gb9b0TJUPG-dOH(_&_xd*f)Elo^wVDCp3`X%O=Qk`nHNRH`u<-m(ob3et-z^LfR)>ByM>h0*}6cE&RQFN51owiGKr!D8+9Cu g4jsjp(S>2mrVgT`Fq4QQX!?(UkU=+Efq$yNZ;?dLTmS$7 diff --git a/src/rarelink/.DS_Store b/src/rarelink/.DS_Store deleted file mode 100755 index f5ac1638dc7b90e08a7ffbb6fa6e4b7ff24873a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKy-ve05I&O#BC&MrggmkL4MG+6N}brMwt`kllnTL~35ki3i6>y<6&QFQ#%_G) zyVAG~j0nM-&@E)5o{7wsroEb+u?^O-Bf$L%N}wE@?qa zx{g*&o*s7X?X@pnU)KG*uy&NAs<%7%$|u^#(S-M5Vhk7q#(**KKN-NC&5|7pT5k*( z1IEB71AIPsG{#)95e!EMy70^grcejLm}?1+@rt=(BM1w`NhnZ4ot79*!lC!-mn$}c z5>8GF45uq^iWe7FM}8mN$>oC98w18b%s?X39^e1x_+>ha{5ZsB#(**K#~5&FF)oJq zD7{S5Z-O8X(&Pu3Oz1(EtppPDPBUXFJMFuDls8JgE3p0{v1jnXMG``#OHBl zcdL}zgBOu91GC@m>`a#VChQMmjCKg%bSioYI zu*r%y8U7;!Si1&Ps|$6*+I`or6HSXjl*tG@-lIX579+>`;FW4^YumDHt6|;yPjc#y z!qK?!!r?8BuB6POML&$L(_|Vnch6-$3e!9ps(>_3z~$~H&0{(B+pgO| zvo)KYI7e;KaeDK%nDq{wwm9t`&*yf1Z~x%zGI-8ja`~z$82B{^7iRb^t`uG2G5RHfdVql{f;I+Qr_aG&6wyq?HXRQEz07XH+QsY+$bktD{ fzIYV3K$U=9Ui`L8!W`?p|Ww+0RdMToaLL3_EqA8WEM@jHNY%3C8{GW40tM6`*lrjOdsm zx}j5w$Fj9S6;K8Kn*zLdt2Ce?c5F)d{hLysqJFpDY$tK*X|HV0&W@8XZ1s~CvdWvE zqt~bXm)&_4onKa@=XpjJ!_z95_uvs2_pp|Tu8O=lX;N06Z{>dQ@ON zXjn)EaAvcW*F9=f1yli5;9CLyK3F(oAeeb{pAH0f1pt=N>PKY_DxUgpu2Ajom-9{4EUfICB< Date: Wed, 25 Mar 2026 16:41:03 +0100 Subject: [PATCH 02/28] updated eval cohort ppkts --- res/evaluation_cohort/phenopackets/1.json | 117 ++++++++++++--------- res/evaluation_cohort/phenopackets/10.json | 111 +++++++++++-------- res/evaluation_cohort/phenopackets/2.json | 99 ++++++++++------- res/evaluation_cohort/phenopackets/3.json | 87 +++++++++------ res/evaluation_cohort/phenopackets/4.json | 89 ++++++++++------ res/evaluation_cohort/phenopackets/5.json | 93 +++++++++------- res/evaluation_cohort/phenopackets/6.json | 89 ++++++++++------ res/evaluation_cohort/phenopackets/7.json | 89 ++++++++++------ res/evaluation_cohort/phenopackets/8.json | 89 ++++++++++------ res/evaluation_cohort/phenopackets/9.json | 103 ++++++++++-------- 10 files changed, 578 insertions(+), 388 deletions(-) diff --git a/res/evaluation_cohort/phenopackets/1.json b/res/evaluation_cohort/phenopackets/1.json index dd3ae91b8..296aa6d76 100644 --- a/res/evaluation_cohort/phenopackets/1.json +++ b/res/evaluation_cohort/phenopackets/1.json @@ -2,13 +2,13 @@ "id": "1", "subject": { "id": "EvCo1", - "date_of_birth": "2015-01-12T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2015-01-12T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P10Y3M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "MALE", @@ -19,16 +19,19 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0002553", "label": "Highly arched eyebrow" }, "onset": { - "ontology_class": { + "ontologyClass": { "id": "HP:0003577", "label": "Congenital onset (at birth)" } @@ -40,7 +43,7 @@ "label": "Macrotia" }, "onset": { - "ontology_class": { + "ontologyClass": { "id": "HP:0003577", "label": "Congenital onset (at birth)" } @@ -91,7 +94,7 @@ { "type": { "id": "HP:0001256", - "label": "Intellectual disability, mild" + "label": "Mild intellectual disability" }, "modifiers": [ { @@ -147,7 +150,7 @@ "value": 7.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y11M" } @@ -167,7 +170,7 @@ "value": 5.6 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y11M" } @@ -187,7 +190,7 @@ "value": 2.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y11M" } @@ -207,7 +210,7 @@ "value": 15.4 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y2M" } @@ -227,7 +230,7 @@ "value": 5.6 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y2M" } @@ -247,7 +250,7 @@ "value": 1.8 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y2M" } @@ -257,22 +260,22 @@ "interpretations": [ { "id": "EvCo1-interpretation-0", - "progress_status": "SOLVED", + "progressStatus": "SOLVED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CAUSATIVE", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "a64391fcf27c42b8a024ede70f6a1d", - "gene_context": { - "value_id": "HGNC:29508", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CAUSATIVE", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "db379c356ba74dcc9c2e830de9cc06", + "geneContext": { + "valueId": "HGNC:29508", "symbol": "KIDINS220" }, "expressions": [ @@ -281,7 +284,7 @@ "value": "NM_020738.4:c.4269del" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -289,14 +292,14 @@ } }, { - "subject_or_biosample_id": "2", - "interpretation_status": "CAUSATIVE", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "3f94cfc4856f41a6aea467b08cef75", - "gene_context": { - "value_id": "HGNC:7133", + "subjectOrBiosampleId": "2", + "interpretationStatus": "CAUSATIVE", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "fa51bd50b0574842a1046f36d5ddb4", + "geneContext": { + "valueId": "HGNC:7133", "symbol": "KMT2D" }, "expressions": [ @@ -305,7 +308,7 @@ "value": "NM_003482.4:c.16529A>G" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -329,59 +332,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:32:10.903009Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:35:16.075566Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/10.json b/res/evaluation_cohort/phenopackets/10.json index 1ad8bfc81..7d98130b1 100644 --- a/res/evaluation_cohort/phenopackets/10.json +++ b/res/evaluation_cohort/phenopackets/10.json @@ -2,13 +2,13 @@ "id": "10", "subject": { "id": "EvCo10", - "date_of_birth": "2015-11-15T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2015-11-15T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P9Y4M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "FEMALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000400", @@ -113,7 +116,7 @@ "value": 5.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y0M" } @@ -133,7 +136,7 @@ "value": 5.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y0M" } @@ -153,7 +156,7 @@ "value": 0.3 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P5Y0M" } @@ -173,7 +176,7 @@ "value": 4.0 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y0M" } @@ -193,7 +196,7 @@ "value": 7.5 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y0M" } @@ -213,7 +216,7 @@ "value": 0.4 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y0M" } @@ -223,22 +226,22 @@ "interpretations": [ { "id": "EvCo10-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CAUSATIVE", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "ec77fecb0be24b99921c2aebe7a25d", - "gene_context": { - "value_id": "HGNC:7133", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CAUSATIVE", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "687b40dd43184d1ab0d375c89628d6", + "geneContext": { + "valueId": "HGNC:7133", "symbol": "KMT2D" }, "expressions": [ @@ -247,7 +250,7 @@ "value": "NM_003482.4:c.16522-5_16522-2del" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -255,14 +258,14 @@ } }, { - "subject_or_biosample_id": "2", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "8b7d5280a1fa4086813aec8d167b46", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "2", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "42ea2477a94944c0b44db1286844bd", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -271,7 +274,7 @@ "value": "NM_001197104.2:c.3019G>T" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -295,59 +298,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:34:00.293353Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:37:24.542514Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/2.json b/res/evaluation_cohort/phenopackets/2.json index 75a579c6d..c35d804fe 100644 --- a/res/evaluation_cohort/phenopackets/2.json +++ b/res/evaluation_cohort/phenopackets/2.json @@ -2,13 +2,13 @@ "id": "2", "subject": { "id": "EvCo2", - "date_of_birth": "2012-03-04T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2012-03-04T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P13Y1M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "FEMALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000400", @@ -100,7 +103,7 @@ "value": 9.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P8Y2M" } @@ -120,7 +123,7 @@ "value": 3.5 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P8Y2M" } @@ -140,7 +143,7 @@ "value": 7.9 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P8Y2M" } @@ -160,7 +163,7 @@ "value": 7.8 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P9Y2M" } @@ -180,7 +183,7 @@ "value": 5.4 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P9Y2M" } @@ -200,7 +203,7 @@ "value": 6.8 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P9Y2M" } @@ -220,7 +223,7 @@ "value": 10.4 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P10Y2M" } @@ -240,7 +243,7 @@ "value": 5.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P10Y2M" } @@ -260,7 +263,7 @@ "value": 5.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P10Y2M" } @@ -270,22 +273,22 @@ "interpretations": [ { "id": "EvCo2-interpretation-0", - "progress_status": "SOLVED", + "progressStatus": "SOLVED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "ed711951287248d2972c0e360d329d", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "36e64b48ed5c43d48e65d43fbec1ef", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -294,7 +297,7 @@ "value": "NM_001197104.2:c.3019G>T" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -318,59 +321,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:32:28.291053Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:35:37.541221Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/3.json b/res/evaluation_cohort/phenopackets/3.json index 4f94d572d..df386a673 100644 --- a/res/evaluation_cohort/phenopackets/3.json +++ b/res/evaluation_cohort/phenopackets/3.json @@ -2,13 +2,13 @@ "id": "3", "subject": { "id": "EvCo3", - "date_of_birth": "2017-06-17T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2017-06-17T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P7Y9M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "MALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000637", @@ -32,7 +35,7 @@ "label": "Moderate" }, "onset": { - "ontology_class": { + "ontologyClass": { "id": "HP:0003577", "label": "Congenital onset (at birth)" } @@ -168,7 +171,7 @@ "value": 1.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P3Y7M" } @@ -188,7 +191,7 @@ "value": 4.8 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P3Y7M" } @@ -198,22 +201,22 @@ "interpretations": [ { "id": "EvCo3-interpretation-0", - "progress_status": "SOLVED", + "progressStatus": "SOLVED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CANDIDATE", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "d7ae57fd6f1441a795125fad2a332d", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CANDIDATE", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "7c2e781f88d04efeae4d8ed8e3a591", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -222,7 +225,7 @@ "value": "NM_001197104.2:c.3019G>T" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -246,59 +249,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:32:38.350095Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:35:51.818400Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/4.json b/res/evaluation_cohort/phenopackets/4.json index 44fd72165..33d089321 100644 --- a/res/evaluation_cohort/phenopackets/4.json +++ b/res/evaluation_cohort/phenopackets/4.json @@ -2,13 +2,13 @@ "id": "4", "subject": { "id": "EvCo4", - "date_of_birth": "2010-02-20T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2010-02-20T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P15Y1M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "FEMALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000175", @@ -75,7 +78,7 @@ "value": 6.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P11Y4M" } @@ -95,7 +98,7 @@ "value": 4.8 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P11Y4M" } @@ -115,7 +118,7 @@ "value": 14.5 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P12Y10M" } @@ -135,7 +138,7 @@ "value": 5.6 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P12Y10M" } @@ -145,22 +148,22 @@ "interpretations": [ { "id": "EvCo4-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "9c3330eeeaa34d0b822430718543bc", - "gene_context": { - "value_id": "HGNC:7133", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "2b830d4bb6634d41b7134f1b64da70", + "geneContext": { + "valueId": "HGNC:7133", "symbol": "KMT2D" }, "expressions": [ @@ -169,7 +172,7 @@ "value": "NM_003482.4:c.16522-5_16522-2del" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -193,59 +196,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:32:48.269920Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:36:03.321322Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/5.json b/res/evaluation_cohort/phenopackets/5.json index 755def023..cf734d6d6 100644 --- a/res/evaluation_cohort/phenopackets/5.json +++ b/res/evaluation_cohort/phenopackets/5.json @@ -2,13 +2,13 @@ "id": "5", "subject": { "id": "EvCo5", - "date_of_birth": "2014-10-05T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2014-10-05T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P10Y6M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "MALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0001212", @@ -142,7 +145,7 @@ "value": 3.9 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P6Y1M" } @@ -162,7 +165,7 @@ "value": 2.0 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P6Y1M" } @@ -182,7 +185,7 @@ "value": 4.3 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y1M" } @@ -202,7 +205,7 @@ "value": 3.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y1M" } @@ -222,7 +225,7 @@ "value": 5.6 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P8Y1M" } @@ -242,7 +245,7 @@ "value": 1.9 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P8Y1M" } @@ -252,22 +255,22 @@ "interpretations": [ { "id": "EvCo5-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CAUSATIVE", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "e0bbac243c534394a5c7b50e3c3716", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CAUSATIVE", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "2283cf1b1dd148aa9e36fced401814", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -276,7 +279,7 @@ "value": "NM_001197104.2:c.5803-1G>A" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -300,59 +303,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:33:02.850795Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:36:21.222765Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/6.json b/res/evaluation_cohort/phenopackets/6.json index 28608aa51..758ed8abb 100644 --- a/res/evaluation_cohort/phenopackets/6.json +++ b/res/evaluation_cohort/phenopackets/6.json @@ -2,13 +2,13 @@ "id": "6", "subject": { "id": "EvCo6", - "date_of_birth": "2014-10-05T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2014-10-05T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P10Y6M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "FEMALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000508", @@ -70,7 +73,7 @@ "value": 6.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y6M" } @@ -90,7 +93,7 @@ "value": 5.5 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y6M" } @@ -110,7 +113,7 @@ "value": 4.0 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y6M" } @@ -130,7 +133,7 @@ "value": 2.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P7Y6M" } @@ -140,22 +143,22 @@ "interpretations": [ { "id": "EvCo6-interpretation-0", - "progress_status": "SOLVED", + "progressStatus": "SOLVED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "b96fd76458bc408ea35301b3ca579e", - "gene_context": { - "value_id": "HGNC:7133", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "4f0cc3dd0bf04d278bc9dd3a4344a7", + "geneContext": { + "valueId": "HGNC:7133", "symbol": "KMT2D" }, "expressions": [ @@ -164,7 +167,7 @@ "value": "NM_003482.4:c.16529A>G" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -188,59 +191,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:33:12.451062Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:36:31.212565Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/7.json b/res/evaluation_cohort/phenopackets/7.json index 34938ff8c..9fb0eb79d 100644 --- a/res/evaluation_cohort/phenopackets/7.json +++ b/res/evaluation_cohort/phenopackets/7.json @@ -2,13 +2,13 @@ "id": "7", "subject": { "id": "EvCo7", - "date_of_birth": "2016-04-25T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2016-04-25T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P8Y11M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "MALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0002553", @@ -90,7 +93,7 @@ "value": 5.9 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P4Y4M" } @@ -110,7 +113,7 @@ "value": 2.3 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P4Y4M" } @@ -130,7 +133,7 @@ "value": 6.3 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P4Y5M" } @@ -150,7 +153,7 @@ "value": 2.3 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P4Y5M" } @@ -160,22 +163,22 @@ "interpretations": [ { "id": "EvCo7-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "e4bedc38ed1e48fc8edcd14b3ff58b", - "gene_context": { - "value_id": "HGNC:29508", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "9903cc8e1f45403aac905618424ada", + "geneContext": { + "valueId": "HGNC:29508", "symbol": "KIDINS220" }, "expressions": [ @@ -184,7 +187,7 @@ "value": "NM_020738.4:c.4269del" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -208,59 +211,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:33:23.098017Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:36:43.341832Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/8.json b/res/evaluation_cohort/phenopackets/8.json index 56d41b6e6..5fcc56de2 100644 --- a/res/evaluation_cohort/phenopackets/8.json +++ b/res/evaluation_cohort/phenopackets/8.json @@ -2,13 +2,13 @@ "id": "8", "subject": { "id": "EvCo8", - "date_of_birth": "2009-12-03T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2009-12-03T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P15Y4M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "FEMALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0000400", @@ -128,7 +131,7 @@ "value": 5.4 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P11Y5M" } @@ -148,7 +151,7 @@ "value": 0.2 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P11Y5M" } @@ -168,7 +171,7 @@ "value": 6.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P12Y5M" } @@ -188,7 +191,7 @@ "value": 1.1 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P12Y5M" } @@ -198,22 +201,22 @@ "interpretations": [ { "id": "EvCo8-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "LIKELY_PATHOGENIC", - "variation_descriptor": { - "id": "2e6f601474b24340b30f8403ac308e", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "LIKELY_PATHOGENIC", + "variationDescriptor": { + "id": "962bfde7a17840838efd3a86d08339", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -222,7 +225,7 @@ "value": "NM_001197104.2:c.3019G>T" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000458", "label": "Heterozygous" } @@ -246,59 +249,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:33:35.095868Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:36:57.740357Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file diff --git a/res/evaluation_cohort/phenopackets/9.json b/res/evaluation_cohort/phenopackets/9.json index fd8e9e5b9..216969096 100644 --- a/res/evaluation_cohort/phenopackets/9.json +++ b/res/evaluation_cohort/phenopackets/9.json @@ -2,13 +2,13 @@ "id": "9", "subject": { "id": "EvCo9", - "date_of_birth": "2013-08-30T00:00:00Z", - "time_at_last_encounter": { + "dateOfBirth": "2013-08-30T00:00:00Z", + "timeAtLastEncounter": { "age": { "iso8601duration": "P11Y7M" } }, - "vital_status": { + "vitalStatus": { "status": "ALIVE" }, "sex": "MALE", @@ -19,9 +19,12 @@ "taxonomy": { "id": "NCBITaxon:9606", "label": "Homo sapiens" + }, + "vital_status": { + "status": "ALIVE" } }, - "phenotypic_features": [ + "phenotypicFeatures": [ { "type": { "id": "HP:0002553", @@ -107,7 +110,7 @@ "value": 8.0 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P9Y6M" } @@ -127,7 +130,7 @@ "value": 2.6 } }, - "time_observed": { + "timeObserved": { "age": { "iso8601duration": "P9Y6M" } @@ -137,22 +140,22 @@ "interpretations": [ { "id": "EvCo9-interpretation-0", - "progress_status": "COMPLETED", + "progressStatus": "COMPLETED", "diagnosis": { "disease": { "id": "MONDO:0007843", "label": "Kabuki syndrome 1" }, - "genomic_interpretations": [ + "genomicInterpretations": [ { - "subject_or_biosample_id": "1", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "05340f24cb68423f885cc46114a84b", - "gene_context": { - "value_id": "HGNC:7132", + "subjectOrBiosampleId": "1", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "983603641ff14b63bb3d460eca54d2", + "geneContext": { + "valueId": "HGNC:7132", "symbol": "KMT2A" }, "expressions": [ @@ -161,7 +164,7 @@ "value": "NM_001197104.2:c.3019G>T" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -169,14 +172,14 @@ } }, { - "subject_or_biosample_id": "2", - "interpretation_status": "CONTRIBUTORY", - "variant_interpretation": { - "acmg_pathogenicity_classification": "PATHOGENIC", - "variation_descriptor": { - "id": "9e02bb2e05984371967fb8278f3ff7", - "gene_context": { - "value_id": "HGNC:7133", + "subjectOrBiosampleId": "2", + "interpretationStatus": "CONTRIBUTORY", + "variantInterpretation": { + "acmgPathogenicityClassification": "PATHOGENIC", + "variationDescriptor": { + "id": "22c57da711994c619b81875a69911e", + "geneContext": { + "valueId": "HGNC:7133", "symbol": "KMT2D" }, "expressions": [ @@ -185,7 +188,7 @@ "value": "NM_003482.4:c.16529A>G" } ], - "allelic_state": { + "allelicState": { "id": "GENO:0000402", "label": "Compound heterozygous" } @@ -209,59 +212,75 @@ } } ], - "meta_data": { - "created": "2025-09-23T15:33:45.047014Z", - "created_by": "Adam G.", + "metaData": { + "created": "2026-03-11T16:37:08.305600Z", + "createdBy": "AG", "resources": [ { "id": "mondo", "name": "Monarch Disease Ontology", "url": "https://purl.obolibrary.org/obo/MONDO/", "version": "2025-06-03", - "namespace_prefix": "MONDO", - "iri_prefix": "http://purl.obolibrary.org/obo/MONDO_" + "namespacePrefix": "MONDO", + "iriPrefix": "http://purl.obolibrary.org/obo/MONDO_" }, { "id": "hpo", "name": "Human Phenotype Ontology", "url": "http://purl.obolibrary.org/obo/hp.owl", "version": "2025-05-06", - "namespace_prefix": "HPO", - "iri_prefix": "http://purl.obolibrary.org/obo/HP_" + "namespacePrefix": "HPO", + "iriPrefix": "http://purl.obolibrary.org/obo/HP_" }, { "id": "loinc", "name": "Logical Observation Identifiers Names and Codes", "url": "https://loinc.org", "version": "LNC278", - "namespace_prefix": "LOINC", - "iri_prefix": "http://loinc.org" + "namespacePrefix": "LOINC", + "iriPrefix": "http://loinc.org" + }, + { + "id": "uo", + "name": "Units of Measurement Ontology", + "url": "https://www.ontobee.org/ontology/UO", + "version": "2023-05-25", + "namespacePrefix": "UO", + "iriPrefix": "http://purl.obolibrary.org/obo/UO_" }, { "id": "hgnc", "name": "HUGO Gene Nomenclature Committee", "url": "https://www.genenames.org/", "version": "2024-08-23", - "namespace_prefix": "HGNC", - "iri_prefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" + "namespacePrefix": "HGNC", + "iriPrefix": "https://www.genenames.org/data/gene-symbol-report/#!/hgnc_id/" }, { "id": "hgvs", "name": "Human Genome Variation Society", "url": "https://varnomen.hgvs.org/", "version": "21.0.0", - "namespace_prefix": "HGVS", - "iri_prefix": "https://varnomen.hgvs.org/recommendations/variant/" + "namespacePrefix": "HGVS", + "iriPrefix": "https://varnomen.hgvs.org/recommendations/variant/" + }, + { + "id": "ga4gh", + "name": "Global Alliance for Genomics and Health", + "url": "https://www.ga4gh.org/", + "version": "v2.0", + "namespacePrefix": "GA4GH", + "iriPrefix": "https://www.ga4gh.org/" }, { "id": "geno", "name": "GENO - The Genotype Ontology", "url": "https://www.genoontology.org/", "version": "2023-10-08", - "namespace_prefix": "GENO", - "iri_prefix": "http://purl.obolibrary.org/obo/GENO_" + "namespacePrefix": "GENO", + "iriPrefix": "http://purl.obolibrary.org/obo/GENO_" } ], - "phenopacket_schema_version": "2.0" + "phenopacketSchemaVersion": "2.0" } } \ No newline at end of file From 4cb33a8e98e4a15ffa8fe226548cb2884be4361b Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Wed, 25 Mar 2026 16:41:12 +0100 Subject: [PATCH 03/28] fixed camelCase in write ppkts --- src/rarelink/phenopackets/write.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rarelink/phenopackets/write.py b/src/rarelink/phenopackets/write.py index 14b7a6595..070064718 100644 --- a/src/rarelink/phenopackets/write.py +++ b/src/rarelink/phenopackets/write.py @@ -3,10 +3,16 @@ from google.protobuf.json_format import MessageToDict from phenopackets import VitalStatus as VitalStatusEnum -def write_phenopackets(phenopackets: list, output_dir: str): +def write_phenopackets(phenopackets: list, output_dir: str, validate: bool = True): """ Writes Phenopackets to JSON files, emitting only the `status` field (including default) in the `vital_status` block. + + Args: + - phenopackets (list): List of Phenopacket protobuf objects. + - output_dir (str): Directory to write JSON files into. + - validate (bool): Whether to validate each phenopacket after writing. + Defaults to True. """ output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) @@ -15,7 +21,7 @@ def write_phenopackets(phenopackets: list, output_dir: str): # 1) Serialize entire phenopacket normally (no default fields) full = MessageToDict( phenopacket, - preserving_proto_field_name=True, + preserving_proto_field_name=False, including_default_value_fields=False ) From 8d1939fb8be7632ba1be90d86e991b7e72229550 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Thu, 26 Mar 2026 12:11:35 +0100 Subject: [PATCH 04/28] issue 205, imrpove ppkts export pipeline, validation, fixed bug --- .gitignore | 3 + src/rarelink/cli/phenopackets/export.py | 442 +++++++++++++++++------- src/rarelink/phenopackets/pipeline.py | 191 ++++++---- src/rarelink/phenopackets/validate.py | 222 +++++++++--- src/rarelink/phenopackets/write.py | 74 +++- 5 files changed, 672 insertions(+), 260 deletions(-) diff --git a/.gitignore b/.gitignore index 5ecae4886..940f0a9ec 100755 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,9 @@ templates/ # Ignore template files from Public folder Public/ +# ignore dev folders +dev/ + # Temporary REDCap test config *.rarelink_test_redcap_config.json diff --git a/src/rarelink/cli/phenopackets/export.py b/src/rarelink/cli/phenopackets/export.py index 09181d7f8..e48d0e9ec 100644 --- a/src/rarelink/cli/phenopackets/export.py +++ b/src/rarelink/cli/phenopackets/export.py @@ -1,35 +1,66 @@ # src/rarelink/cli/phenopackets/export.py import json -import typer +import logging import os -import importlib.util import importlib.machinery from pathlib import Path -from typing import Optional -import logging -from rarelink.rarelink_cdm.mappings import phenopackets as default_mappings +from typing import Callable, Optional +import typer +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table +from rich.text import Text + +from rarelink.rarelink_cdm.mappings import phenopackets as default_mappings from rarelink.cli.utils.terminal_utils import ( between_section_separator, - end_of_section_separator + end_of_section_separator, ) from rarelink.cli.utils.string_utils import ( - success_text, error_text, format_command, - format_header -) -from rarelink.cli.utils.validation_utils import ( - validate_env + format_header, + success_text, ) - +from rarelink.cli.utils.validation_utils import validate_env app = typer.Typer() +console = Console() ENV_PATH = Path(".env") DEFAULT_INPUT_DIR = Path.home() / "Downloads" / "rarelink_records" DEFAULT_OUTPUT_DIR = Path.home() / "Downloads" + +def _make_progress(description: str, total: int) -> tuple: + """ + Create a Progress instance with a single labelled task. + Returns (progress, task_id) ready to use as a context manager. + """ + progress = Progress( + SpinnerColumn(), + TextColumn("[bold cyan]{task.description}"), + BarColumn(bar_width=36), + MofNCompleteColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + console=console, + transient=False, + ) + task_id = progress.add_task(description, total=total) + return progress, task_id + + @app.command() def export( input_path: Path = typer.Option( @@ -39,10 +70,12 @@ def export( None, "--output-dir", "-o", help="Directory to save Phenopackets" ), mappings: Path = typer.Option( - None, "--mappings", "-m", help="Path to custom mapping configuration module" + None, "--mappings", "-m", + help="Path to custom mapping configuration module" ), label_dict: Path = typer.Option( - None, "--label-dict", help="Path to JSON file with code→label mappings" + None, "--label-dict", + help="Path to JSON file with code→label mappings" ), debug: bool = typer.Option( False, "--debug", "-d", help="Enable debug mode for verbose logging" @@ -54,33 +87,32 @@ def export( None, "--created-by", help="Override CREATED_BY from .env" ), bioportal_api_token: Optional[str] = typer.Option( - None, "--bioportal-api-token", help="Provide BioPortal API token (overrides .env)" + None, "--bioportal-api-token", + help="Provide BioPortal API token (overrides .env)" ), timeout: int = typer.Option( 3600, "--timeout", "-t", help="Timeout in seconds (default: 3600)" ), ): """ - CLI command to export data to a cohort of Phenopackets. - - Enhanced to support different data models through custom mapping configurations. + Export REDCap records to a cohort of GA4GH Phenopackets (v2). """ - # Configure logging - log_level = logging.DEBUG if debug else logging.INFO + log_level = logging.DEBUG if debug else logging.WARNING logging.basicConfig(level=log_level) + logging.getLogger("rarelink").setLevel(log_level) logger = logging.getLogger("rarelink.cli.phenopackets.export") format_header("REDCap to Phenopackets Export") - # Step 1: Validate setup files (only if not skipped) + # ── Step 1: Environment validation ────────────────────────────────────── if not skip_validation: typer.echo("🔄 Validating setup files...") typer.echo("🔄 Validating the .env file...") - # Determine which env keys are required required_env_vars = ["CREATED_BY"] - # Only require BioPortal when we actually plan to use it - will_use_bioportal = (bioportal_api_token or os.getenv("BIOPORTAL_API_TOKEN")) and not label_dict + will_use_bioportal = ( + bioportal_api_token or os.getenv("BIOPORTAL_API_TOKEN") + ) and not label_dict if will_use_bioportal: required_env_vars.append("BIOPORTAL_API_TOKEN") @@ -97,40 +129,40 @@ def export( fg=typer.colors.RED, ) typer.secho( - "💡 You can use --skip-validation to bypass environment validation.", + "💡 You can use --skip-validation to bypass this.", fg=typer.colors.YELLOW, ) raise typer.Exit(1) - # Fetch CREATED_BY from env or argument _created_by = created_by or os.getenv("CREATED_BY") if not _created_by and not skip_validation: typer.secho( - error_text("❌ Missing CREATED_BY environment variable or --created-by argument."), + error_text( + "❌ Missing CREATED_BY environment variable or --created-by argument." + ), fg=typer.colors.RED, ) raise typer.Exit(1) - # Fetch BIOPORTAL_API_TOKEN from argument or env _api_token = bioportal_api_token or os.getenv("BIOPORTAL_API_TOKEN") if not _api_token and not skip_validation: typer.secho( - error_text("❌ Missing BioPortal API token. Provide --bioportal-api-token or set BIOPORTAL_API_TOKEN in .env."), + error_text( + "❌ Missing BioPortal API token. " + "Provide --bioportal-api-token or set BIOPORTAL_API_TOKEN in .env." + ), fg=typer.colors.RED, ) raise typer.Exit(1) - # Ensure the BIOPORTAL_API_TOKEN is set in the environment for downstream usage if _api_token: os.environ["BIOPORTAL_API_TOKEN"] = _api_token - # Step 2: Determine input file path + # ── Step 2: Input file ─────────────────────────────────────────────────── if input_path is None: input_path = typer.prompt( - "Enter the path to the validated linkml-json file", - type=Path + "Enter the path to the validated linkml-json file", type=Path ) - if not input_path.exists(): typer.secho( error_text(f"❌ Input file not found: {input_path}."), @@ -138,78 +170,67 @@ def export( ) raise typer.Exit(1) - # Step 3: Determine output directory + # ── Step 3: Output directory ───────────────────────────────────────────── if output_dir is None: - # Try to infer a suitable output directory name - input_stem = input_path.stem - suggested_dir = Path.cwd() / f"{input_stem}_phenopackets" - + suggested_dir = Path.cwd() / f"{input_path.stem}_phenopackets" typer.echo(f"📂 Suggested output directory: {suggested_dir}") - is_correct_output_dir = typer.confirm("Do you want to use this directory?") - if not is_correct_output_dir: + if typer.confirm("Do you want to use this directory?"): + output_dir = suggested_dir + else: output_dir = typer.prompt( - "Enter the path to save Phenopackets", - type=Path + "Enter the path to save Phenopackets", type=Path ) - else: - output_dir = suggested_dir output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) between_section_separator() - # Step 4: Determine mapping configuration + # ── Step 4: Mapping configuration ─────────────────────────────────────── mapping_configs = None if mappings: try: - logger.info(f"Loading custom mappings from: {mappings}") - - # Method 1: Try loading as a regular module - if str(mappings).endswith('.py'): - # Calculate the module name from the file path - module_name = mappings.stem - - # Use importlib machinery to load the module - loader = importlib.machinery.SourceFileLoader(module_name, str(mappings)) - custom_mappings_module = loader.load_module() - - # Check if the module has the expected function - if hasattr(custom_mappings_module, 'create_phenopacket_mappings'): - mapping_configs = custom_mappings_module.create_phenopacket_mappings() - logger.info("Successfully loaded custom mappings") + logger.debug(f"Loading custom mappings from: {mappings}") + if str(mappings).endswith(".py"): + loader = importlib.machinery.SourceFileLoader( + mappings.stem, str(mappings) + ) + mod = loader.load_module() + if hasattr(mod, "create_phenopacket_mappings"): + mapping_configs = mod.create_phenopacket_mappings() else: - logger.warning("No create_phenopacket_mappings function found in module") - # Method 2: Try loading as a JSON file - elif str(mappings).endswith('.json'): - with open(mappings, 'r') as f: + logger.warning( + "No create_phenopacket_mappings function found in module" + ) + elif str(mappings).endswith(".json"): + with open(mappings, "r") as f: mapping_configs = json.load(f) - logger.info("Successfully loaded mappings from JSON file") else: typer.secho( - error_text("❌ Unsupported mapping file format. Use .py or .json files."), - fg=typer.colors.RED + error_text( + "❌ Unsupported mapping file format. Use .py or .json." + ), + fg=typer.colors.RED, ) raise typer.Exit(1) - except Exception as e: typer.secho( error_text(f"❌ Failed to load custom mappings: {str(e)}"), - fg=typer.colors.RED + fg=typer.colors.RED, ) raise typer.Exit(1) else: - try_default = typer.confirm( - "No custom mappings provided. Would you like to try with default RareLink-CDM mappings?" - ) - if try_default: + if typer.confirm( + "No custom mappings provided. Use default RareLink-CDM mappings?" + ): try: - mapping_configs = default_mappings.create_rarelink_phenopacket_mappings() - logger.info("Using default RareLink-CDM mappings") + mapping_configs = ( + default_mappings.create_rarelink_phenopacket_mappings() + ) except Exception as e: typer.secho( error_text( - f"❌ Default RareLink-CDM mappings not available: {str(e)}" + f"❌ Default RareLink-CDM mappings not available: {e}" ), fg=typer.colors.RED, ) @@ -220,83 +241,248 @@ def export( fg=typer.colors.RED, ) raise typer.Exit(1) + if debug: - logger.debug("Using the following mapping configurations:") for key, value in mapping_configs.items(): - logger.debug(f"- {key}: {list(value.keys()) if isinstance(value, dict) else type(value)}") + logger.debug( + f"- {key}: " + f"{list(value.keys()) if isinstance(value, dict) else type(value)}" + ) + # ── Step 5: Optional label dict patch ─────────────────────────────────── if label_dict: - from rarelink.utils.label_fetching import fetch_label as _orig_fetch_label + from rarelink.utils.label_fetching import fetch_label as _orig - # 1) load code→label map with open(label_dict, "r") as lf: label_map = json.load(lf) - # 2) monkey‐patch the pipeline’s fetch_label so it prefers your dict def fetch_label(code: str, enum_class=None, label_dict=None): - # priority: your map → existing logic if code in label_map: return label_map[code] - return _orig_fetch_label(code, enum_class=enum_class, label_dict=label_map) + return _orig(code, enum_class=enum_class, label_dict=label_map) - # 3) inject into the phenopacket pipeline namespace import rarelink.utils.label_fetching as mu mu.fetch_label = fetch_label - - from rarelink.phenopackets import phenopacket_pipeline if not label_dict: - typer.echo("NOTE: This pipeline may fetch labels from BIOPORTAL. " - "Ensure you have an internet connection as this may take a while - time to get a tea ☕ ...") - else: - typer.echo("Using local label dictionary for labels (no BioPortal needed).") - - try: - typer.echo("🚀 Processing your records to Phenopackets...") - - # Load the JSON data from the file - with open(input_path, "r") as f: - input_data = json.load(f) - - # Run the pipeline with enhanced error handling and debug support - phenopackets = phenopacket_pipeline( - input_data=input_data, - output_dir=str(output_dir), - created_by=_created_by, - mapping_configs=mapping_configs, - timeout=timeout, - debug=debug + typer.echo( + "NOTE: This pipeline may fetch labels from BioPortal. " + "Ensure you have an internet connection — time to get a tea ☕ ..." ) + else: + typer.echo("Using local label dictionary (no BioPortal needed).") + + # ── Step 6: Load input data ────────────────────────────────────────────── + with open(input_path, "r") as f: + input_data = json.load(f) + + total = len(input_data) + typer.echo(f"\n🚀 Processing {total} record(s) → Phenopackets\n") - typer.secho(success_text("✅ Phenopackets successfully created!")) - typer.echo(f"📂 Find your Phenopackets here: {output_dir}") + from rarelink.phenopackets.pipeline import phenopacket_pipeline - # Report counts - typer.echo("\nExport Summary:") - typer.echo(f"Total records processed: {len(input_data)}") - typer.echo(f"Total successful exports: {len(phenopackets)}") - typer.echo(f"Total failed exports: {len(input_data) - len(phenopackets)}") + # ── Step 7a: Phase 1 progress bar — Creating ───────────────────────────── + # The validation bar is intentionally absent here. It only appears once + # all phenopackets have been created, making the two-phase flow explicit. + create_progress, create_task = _make_progress( + "Creating phenopackets", total + ) + with create_progress: + def on_created(record_id, success, error): + create_progress.advance(create_task) - # Check for failure report - failure_file = os.path.join(output_dir, "failures.json") - if os.path.exists(failure_file): + try: + result = phenopacket_pipeline( + input_data=input_data, + output_dir=str(output_dir), + created_by=_created_by, + mapping_configs=mapping_configs, + timeout=timeout, + debug=debug, + progress_callback=on_created, + validation_callback=None, # not wired yet — phase 2 below + ) + except Exception as e: typer.secho( - f"⚠️ Some records failed to process. See {failure_file} for details.", - fg=typer.colors.YELLOW + error_text(f"❌ Pipeline failed: {str(e)}"), + fg=typer.colors.RED, ) + if debug: + import traceback + traceback.print_exc() + raise typer.Exit(1) - except Exception as e: - typer.secho( - error_text(f"❌ Failed to export Phenopackets: {str(e)}"), - fg=typer.colors.RED, + # ── Step 7b: Phase 2 progress bar — Validating ─────────────────────────── + # Only appears after the creation bar has finished. Total is n_created + # (records that failed creation have no file to validate). + n_to_validate = result.n_created + if n_to_validate > 0: + validate_progress, validate_task = _make_progress( + "Validating phenopackets", n_to_validate ) - if debug: - import traceback - traceback.print_exc() - raise typer.Exit(1) + with validate_progress: + def on_validated(file_path, success, error): + validate_progress.advance(validate_task) + + # Re-run only the write+validate phase by passing already-created + # phenopackets back through a second pipeline call. + # To avoid re-creating, we call the internal write+validate loop + # directly rather than the full pipeline. + _run_write_and_validate( + phenopackets=result.phenopackets, + output_dir=output_dir, + result=result, + validation_callback=on_validated, + debug=debug, + ) + + # ── Step 8: Summary ────────────────────────────────────────────────────── + console.print() + _print_summary( + total=total, + n_created=result.n_created, + n_failed_creation=result.n_failed_creation, + n_validated=result.n_created - result.n_failed_validation, + n_failed_validation=result.n_failed_validation, + output_dir=output_dir, + failed_creations=result.failed_creations, + failed_validations=result.failed_validations, + ) end_of_section_separator() +def _run_write_and_validate( + phenopackets: list, + output_dir: Path, + result, + validation_callback: Optional[Callable] = None, + debug: bool = False, +): + """ + Write and validate a list of already-created Phenopackets. + Updates result.failed_validations in place. + Called by export() after the creation phase completes. + """ + import json as _json + from google.protobuf.json_format import MessageToDict + from phenopackets import VitalStatus as VitalStatusEnum + from rarelink.phenopackets.validate import validate_phenopackets + + output_path = Path(output_dir) + + for phenopacket in phenopackets: + full = MessageToDict( + phenopacket, + preserving_proto_field_name=False, + including_default_value_fields=False, + ) + raw_status_int = phenopacket.subject.vital_status.status + try: + status_name = VitalStatusEnum.Status.Name(raw_status_int) + except Exception: + status_name = "UNKNOWN_STATUS" + if "subject" in full: + full["subject"]["vitalStatus"] = {"status": status_name} + + file_path = output_path / f"{phenopacket.id}.json" + with open(file_path, "w") as f: + _json.dump(full, f, indent=2) + + try: + ok, details = validate_phenopackets(file_path) + if ok: + if validation_callback: + validation_callback(str(file_path), success=True, error=None) + else: + result.failed_validations.append( + {"file": str(file_path), "error": details} + ) + if validation_callback: + validation_callback(str(file_path), success=False, error=details) + except Exception as e: + error_msg = str(e) + result.failed_validations.append( + {"file": str(file_path), "error": error_msg} + ) + if validation_callback: + validation_callback(str(file_path), success=False, error=error_msg) + + +def _print_summary( + total: int, + n_created: int, + n_failed_creation: int, + n_validated: int, + n_failed_validation: int, + output_dir: Path, + failed_creations: list, + failed_validations: list, +): + """Render a clean Rich summary table with optional failure details.""" + all_ok = (n_failed_creation == 0 and n_failed_validation == 0) + + table = Table( + title="Export Summary", + show_header=True, + header_style="bold white", + title_style="bold cyan", + border_style="cyan", + min_width=52, + ) + table.add_column("Stage", style="dim", width=24) + table.add_column("Passed", justify="right", style="green") + table.add_column("Failed", justify="right") + table.add_column("Total", justify="right", style="bold") + + def _fail_style(n: int) -> Text: + return Text(str(n), style="bold red" if n > 0 else "green") + + table.add_row("Creation", str(n_created), _fail_style(n_failed_creation), str(total)) + table.add_row("Validation", str(n_validated), _fail_style(n_failed_validation), str(n_created)) + + console.print(table) + console.print(f" 📂 Output directory: [bold]{output_dir}[/bold]\n") + + if all_ok: + console.print( + Panel( + "✅ [bold green]All phenopackets created and validated successfully![/bold green]", + border_style="green", + ) + ) + return + + if failed_creations: + console.print() + fail_table = Table( + title=f"❌ Creation Failures ({len(failed_creations)})", + show_header=True, header_style="bold red", + title_style="bold red", border_style="red", + ) + fail_table.add_column("Record ID", style="bold", width=16) + fail_table.add_column("Error") + for f in failed_creations: + fail_table.add_row(str(f["record_id"]), f["error"]) + console.print(fail_table) + + if failed_validations: + console.print() + val_table = Table( + title=f"⚠️ Validation Failures ({len(failed_validations)})", + show_header=True, header_style="bold yellow", + title_style="bold yellow", border_style="yellow", + ) + val_table.add_column("File", style="bold", width=32) + val_table.add_column("Error") + for f in failed_validations: + val_table.add_row(Path(f["file"]).name, f["error"]) + console.print(val_table) + + failure_file = output_dir / "failures.json" + if failure_file.exists(): + console.print(f"\n 💾 Full failure report: [bold]{failure_file}[/bold]") + + if __name__ == "__main__": - app() + app() \ No newline at end of file diff --git a/src/rarelink/phenopackets/pipeline.py b/src/rarelink/phenopackets/pipeline.py index 8e5a8756a..5918617d3 100755 --- a/src/rarelink/phenopackets/pipeline.py +++ b/src/rarelink/phenopackets/pipeline.py @@ -1,123 +1,162 @@ # src/rarelink/phenopackets/pipeline.py -import typer -from pathlib import Path -from typing import Dict, Any, Optional -import logging -import signal import json +import logging import os +import signal +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +import typer -from rarelink.phenopackets import ( - create_phenopacket -) +from rarelink.phenopackets import create_phenopacket +from rarelink.phenopackets.write import write_phenopackets app = typer.Typer() DEFAULT_OUTPUT_DIR = Path.home() / "Downloads" / "phenopackets" logger = logging.getLogger(__name__) + class TimeoutException(Exception): pass + def timeout_handler(signum, frame): - raise TimeoutException("Pipeline processing exceeded the one-hour timeout limit.") + raise TimeoutException( + "Pipeline processing exceeded the timeout limit." + ) + + +@dataclass +class PipelineResult: + """Structured result returned by phenopacket_pipeline.""" + phenopackets: list = field(default_factory=list) + failed_creations: List[Dict[str, str]] = field(default_factory=list) + failed_validations: List[Dict[str, str]] = field(default_factory=list) + total_records: int = 0 + + @property + def n_created(self) -> int: + return len(self.phenopackets) + + @property + def n_failed_creation(self) -> int: + return len(self.failed_creations) + + @property + def n_failed_validation(self) -> int: + return len(self.failed_validations) + def phenopacket_pipeline( - input_data: list, - output_dir: str, - created_by: str, + input_data: list, + output_dir: str, + created_by: str, mapping_configs: Optional[Dict[str, Any]] = None, timeout: int = 3600, - debug: bool = False -): + debug: bool = False, + progress_callback: Optional[Callable] = None, + validation_callback: Optional[Callable] = None, +) -> PipelineResult: """ - Enhanced pipeline to process input data, create Phenopackets, and write them to files. - Now handles different data models through flexible mapping configurations. + Process input records into Phenopackets in two explicit phases: + + Phase 1 — Create: all records are mapped and built in memory. + Phase 2 — Write & Validate: write_phenopackets() serializes each one + to disk and optionally validates it. Args: - input_data (list): List of dictionaries containing individual records. - output_dir (str): Directory to save Phenopacket JSON files. - created_by (str): Name of the creator (for metadata). - mapping_configs (dict, optional): Mapping configurations for Phenopacket creation. - timeout (int): Timeout in seconds (default is 3600 seconds = 1 hour). - debug (bool): Enable debug mode for verbose logging + input_data: List of record dicts. + output_dir: Directory to write JSON files into. + created_by: Creator name for phenopacket metadata. + mapping_configs: Mapping configurations for Phenopacket creation. + timeout: Wall-clock timeout in seconds (default 3600). + debug: Enable verbose debug logging. + progress_callback: Optional callable(record_id, success, error) + called after each creation attempt. + validation_callback: Optional callable(file_path, success, error) + called after each validation attempt. + Forwarded directly to write_phenopackets(). Returns: - List: A list of created Phenopacket objects. + PipelineResult with created phenopackets and per-stage failure details. """ - # Set up logging level based on debug flag - if debug: - logging.getLogger('rarelink').setLevel(logging.DEBUG) - else: - logging.getLogger('rarelink').setLevel(logging.INFO) + logging.getLogger("rarelink").setLevel( + logging.DEBUG if debug else logging.WARNING + ) - # Set up the alarm signal for the timeout signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(timeout) + result = PipelineResult(total_records=len(input_data)) + try: - # Validate mapping_configs if not mapping_configs: raise ValueError("Mapping configurations are required") - - # Create directory if it doesn't exist - os.makedirs(output_dir, exist_ok=True) - # Create Phenopackets - phenopackets = [] - failed_records = [] - total_records = len(input_data) + os.makedirs(output_dir, exist_ok=True) - for i, record in enumerate(input_data): + # ── Phase 1: Create all Phenopackets ────────────────────────────────── + for record in input_data: + record_id = record.get("record_id", "unknown") try: - print(f"Processing record {i+1}/{total_records} (id={record.get('record_id', 'unknown')})") - - # Use mapping_configs if provided phenopacket = create_phenopacket( - data=record, + data=record, created_by=created_by, mapping_configs=mapping_configs, - debug=debug + debug=debug, ) - - phenopackets.append(phenopacket) - print(f" ... created Phenopacket for record id={record.get('record_id', 'unknown')}") + result.phenopackets.append(phenopacket) + if progress_callback: + progress_callback(record_id, success=True, error=None) except Exception as e: - print(f"ERROR creating Phenopacket for record id={record.get('record_id', 'unknown')} - {e}") - failed_records.append({ - 'record_id': record.get('record_id', 'unknown'), - 'error': str(e) - }) - - # Log additional debug info + error_msg = str(e) + result.failed_creations.append( + {"record_id": record_id, "error": error_msg} + ) + if progress_callback: + progress_callback(record_id, success=False, error=error_msg) if debug: - logger.debug(f"Record structure: {json.dumps(record, default=str, indent=2)[:1000]}...") - - # Write Phenopackets to files - from rarelink.phenopackets import write_phenopackets - logger.info("Writing Phenopackets to files...") - write_phenopackets(phenopackets, output_dir) - logger.info("Phenopacket pipeline completed successfully.") - - # Optionally, log details of failed records - if failed_records: - logger.warning("Details of failed records:") - for fail in failed_records: - logger.warning(f"Record ID: {fail['record_id']}") - logger.warning(f"Error: {fail['error']}") - - # Write failure report to file for easier debugging + logger.debug( + "Record structure: " + f"{json.dumps(record, default=str, indent=2)[:1000]}..." + ) + + # ── Phase 2: Write & Validate ───────────────────────────────────────── + # Wrap the validation_callback so we can also capture failures into + # result.failed_validations for the summary report. + def _validation_callback(file_path: str, success: bool, error: Optional[str]): + if not success: + result.failed_validations.append( + {"file": file_path, "error": error or ""} + ) + if validation_callback: + validation_callback(file_path, success=success, error=error) + + write_phenopackets( + phenopackets=result.phenopackets, + output_dir=output_dir, + validate=True, + validation_callback=_validation_callback, + ) + + # ── Write combined failure report ────────────────────────────────────── + all_failures = [ + {**f, "stage": "creation"} for f in result.failed_creations + ] + [ + {**f, "stage": "validation"} for f in result.failed_validations + ] + if all_failures: failure_file = os.path.join(output_dir, "failures.json") - with open(failure_file, 'w') as f: - json.dump(failed_records, f, indent=2) - logger.info(f"Failure report written to {failure_file}") + with open(failure_file, "w") as fh: + json.dump(all_failures, fh, indent=2) + logger.debug(f"Failure report written to {failure_file}") + + return result - return phenopackets - except TimeoutException as te: logger.error(f"Timeout occurred: {te}") - print(f"WARNING: Processing timed out after {timeout/3600} hour(s).") raise finally: - # Disable the alarm signal.alarm(0) \ No newline at end of file diff --git a/src/rarelink/phenopackets/validate.py b/src/rarelink/phenopackets/validate.py index 21094e8d9..6e9fa62d9 100644 --- a/src/rarelink/phenopackets/validate.py +++ b/src/rarelink/phenopackets/validate.py @@ -1,67 +1,204 @@ +import json +import re +import shutil import subprocess from pathlib import Path -from typing import Tuple, List, Union +from typing import List, Tuple, Union import logging logger = logging.getLogger(__name__) -def validate_phenopackets(path: Path) -> Union[Tuple[bool, str], List[Tuple[bool, str]]]: +# --------------------------------------------------------------------------- +# Required top-level keys for a Phenopacket v2 JSON document +# --------------------------------------------------------------------------- +_REQUIRED_TOP_LEVEL = {"id", "metaData"} + +# Required keys inside metaData +_REQUIRED_METADATA = {"created", "createdBy", "phenopacketSchemaVersion"} + +# Basic pattern for CURIE-style ontology term IDs (e.g. HP:0001250, MONDO:0007843) +_CURIE_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_]*:[A-Za-z0-9._\-]+$") + + +def validate_phenopackets( + path: Path, +) -> Union[Tuple[bool, str], List[Tuple[bool, str]]]: """ - Validates a phenopacket file or directory of phenopackets using the `phenopacket-tools` CLI. + Validates a phenopacket file or directory of phenopackets. + + Validation is performed in two stages: + 1. **Python-native checks** — always available, no external tooling needed. + Verifies JSON structure, required fields, and ontology term format. + 2. **phenopacket-tools CLI** — used automatically when the CLI is on PATH, + providing deeper schema-level validation on top of the Python checks. Args: - path (Path): Path to a phenopacket file or directory of phenopackets. + path (Path): Path to a single ``.json`` file or a directory of them. Returns: - Union[Tuple[bool, str], List[Tuple[bool, str]]]: Validation results. - - Single file: (bool, str) - Success/Failure and details. - - Directory: List of (bool, str) for each file. - + - Single file → ``(bool, str)`` — (passed, detail message) + - Directory → ``List[(bool, str)]`` — one tuple per file + Raises: - ValueError: If the path is invalid or contains no valid phenopackets. + ValueError: If the path does not exist, is not a JSON file, or a + directory contains no JSON files. """ logger.info("Starting validation of phenopackets...") if not path.exists(): raise ValueError(f"Path {path} does not exist.") - + if path.is_file(): if path.suffix == ".json": return _validate_single_phenopacket(path) - else: - raise ValueError(f"File {path} is not a valid JSON file.") - elif path.is_dir(): - results = [] - for file_path in path.glob("*.json"): - results.append(_validate_single_phenopacket(file_path)) - + raise ValueError(f"File {path} is not a valid JSON file.") + + if path.is_dir(): + results = [ + _validate_single_phenopacket(fp) for fp in sorted(path.glob("*.json")) + ] if not results: - raise ValueError(f"Directory {path} does not contain any JSON files.") - - logger.info(f"Validation completed: {len(results)} files validated.") + raise ValueError(f"Directory {path} contains no JSON files.") + passed = sum(1 for ok, _ in results if ok) + logger.info( + f"Validation completed: {passed}/{len(results)} files passed." + ) return results - else: - raise ValueError(f"Path {path} is neither a file nor a directory.") + + raise ValueError(f"Path {path} is neither a file nor a directory.") + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- def _validate_single_phenopacket(file_path: Path) -> Tuple[bool, str]: + """Run all available validation stages on one file.""" + logger.info(f"Validating {file_path}...") + + # --- Stage 1: Python-native structural validation --- + ok, msg = _python_validate(file_path) + if not ok: + logger.error(f"Structural validation failed for {file_path}: {msg}") + return False, msg + + # --- Stage 2: phenopacket-tools CLI (optional) --- + if shutil.which("phenopacket-tools"): + ok, msg = _cli_validate(file_path) + if not ok: + logger.warning(f"CLI validation failed for {file_path}: {msg}") + return False, msg + logger.info(f"CLI validation passed for {file_path}") + else: + logger.debug( + "phenopacket-tools CLI not found on PATH — skipping CLI validation. " + "Install it for deeper schema checks: " + "https://github.com/phenopackets/phenopacket-tools" + ) + + logger.info(f"Validation passed: {file_path}") + return True, f"OK: {file_path}" + + +def _python_validate(file_path: Path) -> Tuple[bool, str]: """ - Validates a single phenopacket using the `phenopacket-tools` CLI. + Lightweight Python-native checks against the Phenopacket v2 JSON schema: - Args: - file_path (Path): Path to a single phenopacket file. + - Valid, parseable JSON + - Required top-level keys present (``id``, ``metaData``) + - ``metaData`` contains ``created``, ``createdBy``, ``phenopacketSchemaVersion`` + - ``phenopacketSchemaVersion`` is ``"2.0"`` + - All ontology terms (wherever ``{"id": ..., "label": ...}`` objects appear) + have a CURIE-formatted ``id`` + - ``subject.id`` present when ``subject`` block exists + """ + # 1. Parse JSON + try: + with open(file_path) as f: + doc = json.load(f) + except json.JSONDecodeError as exc: + return False, f"Invalid JSON: {exc}" - Returns: - Tuple[bool, str]: Validation result (Success/Failure) and details. + errors: List[str] = [] + + # 2. Required top-level keys + missing = _REQUIRED_TOP_LEVEL - doc.keys() + if missing: + errors.append(f"Missing required top-level keys: {sorted(missing)}") + + # 3. metaData checks + meta = doc.get("metaData", {}) + if isinstance(meta, dict): + missing_meta = _REQUIRED_METADATA - meta.keys() + if missing_meta: + errors.append( + f"metaData missing required keys: {sorted(missing_meta)}" + ) + schema_ver = meta.get("phenopacketSchemaVersion", "") + if schema_ver and not schema_ver.startswith("2"): + errors.append( + f"phenopacketSchemaVersion is '{schema_ver}', expected '2.0'" + ) + + # 4. subject.id + subject = doc.get("subject") + if subject is not None: + if not isinstance(subject, dict) or not subject.get("id"): + errors.append("subject block is present but missing 'id'") + + # 5. Ontology term CURIE format — walk the whole document + curie_errors = _check_curie_terms(doc, path="$") + errors.extend(curie_errors) + + if errors: + detail = "; ".join(errors) + return False, detail + + return True, "Structural checks passed" + + +def _check_curie_terms(node, path: str) -> List[str]: + """ + Recursively walk *node* and collect CURIE-format violations for any + object that has an ``"id"`` key whose sibling is ``"label"`` — i.e. an + OntologyClass-like object. """ - command = f"phenopacket-tools validate {file_path}" + errors: List[str] = [] + + if isinstance(node, dict): + # Looks like an OntologyClass if it has both "id" and "label" + if "id" in node and "label" in node: + term_id = node["id"] + if isinstance(term_id, str) and not _CURIE_PATTERN.match(term_id): + errors.append( + f"Ontology term id '{term_id}' at {path} " + f"is not a valid CURIE (expected format PREFIX:localid)" + ) + for key, value in node.items(): + errors.extend(_check_curie_terms(value, path=f"{path}.{key}")) + + elif isinstance(node, list): + for i, item in enumerate(node): + errors.extend(_check_curie_terms(item, path=f"{path}[{i}]")) + + return errors + + +def _cli_validate(file_path: Path) -> Tuple[bool, str]: + """Run ``phenopacket-tools validate`` and return (success, output).""" + command = ["phenopacket-tools", "validate", str(file_path)] try: - logger.info(f"Validating {file_path}...") - output = subprocess.check_output(command, shell=True, text=True) - logger.info(f"Validation output for {file_path}:\n{output}") + output = subprocess.check_output( + command, stderr=subprocess.STDOUT, text=True + ) return True, output - except subprocess.CalledProcessError as e: - logger.error(f"Validation failed for {file_path}:\n{e.output}") - return False, e.output + except subprocess.CalledProcessError as exc: + return False, exc.output or str(exc) + + +# --------------------------------------------------------------------------- +# CLI entry-point +# --------------------------------------------------------------------------- if __name__ == "__main__": import argparse @@ -73,9 +210,9 @@ def _validate_single_phenopacket(file_path: Path) -> Tuple[bool, str]: description="Validate a phenopacket file or directory of phenopackets." ) parser.add_argument( - "path", - type=Path, - help="Path to the phenopacket file or directory to validate." + "path", + type=Path, + help="Path to the phenopacket file or directory to validate.", ) args = parser.parse_args() @@ -85,8 +222,13 @@ def _validate_single_phenopacket(file_path: Path) -> Tuple[bool, str]: for success, details in results: if not success: logger.error(details) - elif not results[0]: - logger.error(results[1]) + if not all(ok for ok, _ in results): + sys.exit(1) + else: + success, details = results + if not success: + logger.error(details) + sys.exit(1) except ValueError as ve: logger.error(str(ve)) - sys.exit(1) + sys.exit(1) \ No newline at end of file diff --git a/src/rarelink/phenopackets/write.py b/src/rarelink/phenopackets/write.py index 070064718..3a33b55fb 100644 --- a/src/rarelink/phenopackets/write.py +++ b/src/rarelink/phenopackets/write.py @@ -1,44 +1,86 @@ +# src/rarelink/phenopackets/write.py import json +import logging from pathlib import Path +from typing import Callable, Optional + from google.protobuf.json_format import MessageToDict from phenopackets import VitalStatus as VitalStatusEnum -def write_phenopackets(phenopackets: list, output_dir: str, validate: bool = True): +from rarelink.phenopackets.validate import validate_phenopackets + +logger = logging.getLogger(__name__) + + +def write_phenopackets( + phenopackets: list, + output_dir: str, + validate: bool = True, + validation_callback: Optional[Callable] = None, +) -> None: """ - Writes Phenopackets to JSON files, emitting only the `status` field - (including default) in the `vital_status` block. + Serialize Phenopackets to JSON files (camelCase, Phenopacket v2 spec) + and optionally validate each one after writing. Args: - - phenopackets (list): List of Phenopacket protobuf objects. - - output_dir (str): Directory to write JSON files into. - - validate (bool): Whether to validate each phenopacket after writing. - Defaults to True. + phenopackets: List of Phenopacket protobuf objects. + output_dir: Directory to write JSON files into. + validate: Run validation after each write. Defaults to True. + validation_callback: Optional callable(file_path, success, error) + called after each validation attempt. + Allows the pipeline to drive progress bars. """ output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) for phenopacket in phenopackets: - # 1) Serialize entire phenopacket normally (no default fields) full = MessageToDict( phenopacket, preserving_proto_field_name=False, - including_default_value_fields=False + including_default_value_fields=False, ) - # 2) Extract the raw integer enum value + # 2) Extract the raw integer enum value for vital status. raw_status_int = phenopacket.subject.vital_status.status - # 3) Map back to the enum name + # 3) Map back to the enum name string. try: status_name = VitalStatusEnum.Status.Name(raw_status_int) except Exception: status_name = "UNKNOWN_STATUS" - # 4) Overwrite the vital_status block with just {"status": name} - full["subject"]["vital_status"] = {"status": status_name} + # 4) Overwrite the vitalStatus block with just {"status": name}. + if "subject" in full: + full["subject"]["vitalStatus"] = {"status": status_name} - # 5) Write to disk - file_name = f"{phenopacket.id}.json" - file_path = output_path / file_name + # 5) Write to disk. + file_path = output_path / f"{phenopacket.id}.json" with open(file_path, "w") as f: json.dump(full, f, indent=2) + logger.debug(f"Written: {file_path}") + + # 6) Optionally validate after writing. + if validate: + try: + success, details = validate_phenopackets(file_path) + if success: + logger.debug(f"Validation passed: {file_path}") + if validation_callback: + validation_callback( + str(file_path), success=True, error=None + ) + else: + logger.warning( + f"Validation failed for {file_path}: {details}" + ) + if validation_callback: + validation_callback( + str(file_path), success=False, error=details + ) + except Exception as e: + error_msg = str(e) + logger.warning(f"Validation error for {file_path}: {error_msg}") + if validation_callback: + validation_callback( + str(file_path), success=False, error=error_msg + ) \ No newline at end of file From 795d3ea89ad920d23cd48e7814653192ee4669d9 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Thu, 26 Mar 2026 12:19:04 +0100 Subject: [PATCH 05/28] clean up code --- src/rarelink/cli/phenopackets/export.py | 8 -------- src/rarelink/phenopackets/pipeline.py | 3 --- 2 files changed, 11 deletions(-) diff --git a/src/rarelink/cli/phenopackets/export.py b/src/rarelink/cli/phenopackets/export.py index e48d0e9ec..4d3322e89 100644 --- a/src/rarelink/cli/phenopackets/export.py +++ b/src/rarelink/cli/phenopackets/export.py @@ -282,8 +282,6 @@ def fetch_label(code: str, enum_class=None, label_dict=None): from rarelink.phenopackets.pipeline import phenopacket_pipeline # ── Step 7a: Phase 1 progress bar — Creating ───────────────────────────── - # The validation bar is intentionally absent here. It only appears once - # all phenopackets have been created, making the two-phase flow explicit. create_progress, create_task = _make_progress( "Creating phenopackets", total ) @@ -313,8 +311,6 @@ def on_created(record_id, success, error): raise typer.Exit(1) # ── Step 7b: Phase 2 progress bar — Validating ─────────────────────────── - # Only appears after the creation bar has finished. Total is n_created - # (records that failed creation have no file to validate). n_to_validate = result.n_created if n_to_validate > 0: validate_progress, validate_task = _make_progress( @@ -324,10 +320,6 @@ def on_created(record_id, success, error): def on_validated(file_path, success, error): validate_progress.advance(validate_task) - # Re-run only the write+validate phase by passing already-created - # phenopackets back through a second pipeline call. - # To avoid re-creating, we call the internal write+validate loop - # directly rather than the full pipeline. _run_write_and_validate( phenopackets=result.phenopackets, output_dir=output_dir, diff --git a/src/rarelink/phenopackets/pipeline.py b/src/rarelink/phenopackets/pipeline.py index 5918617d3..d5a177ee7 100755 --- a/src/rarelink/phenopackets/pipeline.py +++ b/src/rarelink/phenopackets/pipeline.py @@ -1,4 +1,3 @@ -# src/rarelink/phenopackets/pipeline.py import json import logging import os @@ -124,8 +123,6 @@ def phenopacket_pipeline( ) # ── Phase 2: Write & Validate ───────────────────────────────────────── - # Wrap the validation_callback so we can also capture failures into - # result.failed_validations for the summary report. def _validation_callback(file_path: str, success: bool, error: Optional[str]): if not success: result.failed_validations.append( From 44dc4afaa6627642bda6d91d7404627912dc396c Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Thu, 26 Mar 2026 12:20:24 +0100 Subject: [PATCH 06/28] clean up --- src/rarelink/phenopackets/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rarelink/phenopackets/write.py b/src/rarelink/phenopackets/write.py index 3a33b55fb..58d7a7395 100644 --- a/src/rarelink/phenopackets/write.py +++ b/src/rarelink/phenopackets/write.py @@ -19,7 +19,7 @@ def write_phenopackets( validation_callback: Optional[Callable] = None, ) -> None: """ - Serialize Phenopackets to JSON files (camelCase, Phenopacket v2 spec) + Serialize Phenopackets to JSON files (Phenopacket v2 spec) and optionally validate each one after writing. Args: From 1f276eb5bc633017802c1e48522614b1914681a5 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:49:27 +0200 Subject: [PATCH 07/28] updated rarelink figure in docs --- .../2_1_rarelink_overview.rst | 12 +- docs/_static/res/rarelink_overview.svg | 26571 ---------------- 2 files changed, 6 insertions(+), 26577 deletions(-) delete mode 100644 docs/_static/res/rarelink_overview.svg diff --git a/docs/2_rarelink_framework/2_1_rarelink_overview.rst b/docs/2_rarelink_framework/2_1_rarelink_overview.rst index bf6394f40..d849cce9b 100755 --- a/docs/2_rarelink_framework/2_1_rarelink_overview.rst +++ b/docs/2_rarelink_framework/2_1_rarelink_overview.rst @@ -16,17 +16,17 @@ GA4GH Phenopackets, without the need for additional coding or mapping. As a result, RareLink allows researchers and clinicians to collect and process RD data seamlessly within the REDCap environment. -.. figure:: ../_static/res/rarelink_overview.svg +.. figure:: ../_static/res/rarelink_overview.png Overview of the RareLink framework integrated with a local REDCap instance. - The framework comprises the RareLink instruments based on the RD-CDM, the - RareLink API, the toFHIR & CDIS module for export and import to HL7 FHIR IPS - and Genomic Reporting Resources, and the RareLink-Phenopackets module for + The framework comprises the REDCap instruments based on the `RD-CDM `_, + the RareLink API, the toFHIR & CDIS module for export and import to HL7 FHIR + IPS and Genomic Reporting Resources, and the RareLink-Phenopackets module for direct export to GA4GH Phenopackets. Additionally, the LinkML-based import mapper facilitates data import from tabular databases into REDCap, while the Manual Data Capture Guide assists with the manual data capture of - RD-CDM data in a local REDCap project. Click `here `_ - to see the figure in full size. + RD-CDM data in a local REDCap project. The framework is also configurable + for disease-specific extensions or custom data models. .. note:: You will need your own REDCap instance to deploy the RareLink framework. diff --git a/docs/_static/res/rarelink_overview.svg b/docs/_static/res/rarelink_overview.svg deleted file mode 100644 index f637bd39a..000000000 --- a/docs/_static/res/rarelink_overview.svg +++ /dev/null @@ -1,26571 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From f86d7e3ee6cda89178ce73f6f30cc3a75ee320e7 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:49:40 +0200 Subject: [PATCH 08/28] added variantValidator for manual hgvs curation --- docs/4_user_guide/4_1_manual_data_capture.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/4_user_guide/4_1_manual_data_capture.rst b/docs/4_user_guide/4_1_manual_data_capture.rst index 156f98968..828975288 100755 --- a/docs/4_user_guide/4_1_manual_data_capture.rst +++ b/docs/4_user_guide/4_1_manual_data_capture.rst @@ -354,7 +354,8 @@ Fields: 1. Validate mutations using `ClinVar `_ or `Varsome `_. - 2. Confirm the expression with the `HGVS Validator `_. + 2. Confirm the expression with the `Variant Validator `_ + or the `HGVS Validator `_. 3. If the validation fails, enter the details in the **6.1.6 Genetic Mutation String** field. 4. ``*``: **Are you sure the entered HGVS expression was validated using the variant validator?** - this field is mandatory to ensure the From c66d56394206938a0deb162f6c95eec84065dc25 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:49:50 +0200 Subject: [PATCH 09/28] added phenopacket adapters to docs --- docs/4_user_guide/4_3_phenopackets.rst | 314 ++++++++++++++++++++++--- 1 file changed, 276 insertions(+), 38 deletions(-) diff --git a/docs/4_user_guide/4_3_phenopackets.rst b/docs/4_user_guide/4_3_phenopackets.rst index 6a1726b82..9283e9ed4 100755 --- a/docs/4_user_guide/4_3_phenopackets.rst +++ b/docs/4_user_guide/4_3_phenopackets.rst @@ -21,9 +21,12 @@ ________________________________________________________________________________ - :ref:`phenopacket-validation` - :ref:`rarelink-phenopacket-preconfigurations` - :ref:`phenopackets-other-redcap-data-models` - - :ref:`troubleshooting` + - :ref:`phenopacket-adapters` + - :ref:`multi-onset-adapter` + - :ref:`ontology-routing-adapter` + - :ref:`troubleshooting` _____________________________________________________________________________________ .. _get_started: @@ -686,55 +689,290 @@ placeholders with relevant codes and Phenopacket terms. _____________________________________________________________________________________ -.. _factory-approach: +.. _phenopacket-adapters: -Using the Factory Approach -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Phenopacket Adapters +==================== -For more advanced use cases, you can use the PhenopacketMappingFactory to create -mappings for different data models: +Adapters are preprocessing functions that transform or split data **before** +any mapper runs. They are designed to solve structural challenges that arise +when a single REDCap instrument or data element does not map cleanly to a +single Phenopacket block. All adapters are: -.. code-block:: python +- **Opt-in** — activated by a configuration key; absent means no effect. +- **Mapper-agnostic** — they produce data shaped to the conventions that + existing mappers already understand, so no mapper code changes are needed. +- **Reusable** — they are designed for general use, not tied to any specific + data model. - from rarelink.phenopackets.factory import PhenopacketMappingFactory, get_phenopacket_mappings +The adapters live in ``src/rarelink/phenopackets/adapter/``. - # Get mappings for RareLink CDM (default) - rarelink_mappings = get_phenopacket_mappings() +----- - # Get mappings for a custom data model - custom_mappings = get_phenopacket_mappings(model_name="your_model") +.. _multi-onset-adapter: - # Creating phenopackets with specific model mappings - from rarelink.phenopackets import create_phenopacket +Multi-Onset Adapter +------------------- - # Create phenopacket using the specified mappings - phenopacket = create_phenopacket( - data=record_data, - created_by="Your Name", - mapping_configs=custom_mappings - ) +**Module:** ``rarelink.phenopackets.adapter.multi_onset`` -The factory approach provides additional helper functions for working with different -data models: +**When to use it** -1. **Converting to Multi-Instrument Format** - - .. code-block:: python - - updated_config = PhenopacketMappingFactory.convert_to_multi_instrument_format( - config=mappings, - block_name="phenotypicFeatures" - ) +Use the multi-onset adapter when a single clinical finding has been observed +on **multiple occasions**, each with its own date, and you want to represent +each observation as a separate ``PhenotypicFeature`` entry with its own +``onset`` value. -2. **Merging Configurations** - - .. code-block:: python - - merged_config = PhenopacketMappingFactory.merge_configurations( - base_config=base_mappings, - override_config=custom_overrides - ) +For example, a recurrent respiratory infection may have been recorded on +three separate dates. Without multi-onset, only the first date would be +captured. With multi-onset, three separate ``PhenotypicFeature`` messages +are produced — one per date — all sharing the same type, severity, and +modifiers. + +**Configuration** + +Enable multi-onset in the mapping block for the relevant phenotypicFeatures +instrument:: + + FEATURES_BLOCK = { + "redcap_repeat_instrument": "your_instrument_name", + "type_field": "your_type_field", + "severity_field": "your_severity_field", + + # Enable multi-onset + "multi_onset": True, + "onset_date_fields": [ + "onset_date_1", + "onset_date_2", + "onset_date_3", + # ... add as many as your instrument supports + ], + } + +**What it does** + +For each non-empty date in ``onset_date_fields``: + +1. Creates the base ``PhenotypicFeature`` using the normal mapping function. +2. Deep-copies it. +3. Replaces the ``onset`` field with an ``Age`` element computed from that + date and the subject's date of birth. +4. Appends the copy to the result list. + +If no valid dates are found, the base feature (with whatever onset was +already set) is returned as a single-element list. + +**Data shape expected** + +The adapter reads onset dates directly from the inner instrument dict of +each repeated element:: + + { + "redcap_repeat_instrument": "your_instrument_name", + "redcap_repeat_instance": 1, + "your_instrument_name": { + "your_type_field": "HP:0004469", + "severity_field": "HP:0012826", + "onset_date_1": "2022-02-01", + "onset_date_2": "2023-03-01", + "onset_date_3": "2023-12-01" + } + } + +Produces three separate ``PhenotypicFeature`` messages, each with +``onset.age.iso8601duration`` computed from the respective date and DOB. + +**Notes** + +- All copies share the same type, severity, and modifiers. +- If DOB is unavailable, no age calculation is possible and the base + feature is returned unchanged. +- Combine with :ref:`ontology-routing-adapter` when the same instrument + also mixes HP and MONDO codes. + +----- + +.. _ontology-routing-adapter: + +Ontology Routing Adapter +------------------------- + +**Module:** ``rarelink.phenopackets.adapter.ontology_routing_adapter`` + +**When to use it** + +Use the ontology routing adapter when a **single data element** (or a set +of mutually exclusive sub-fields within one instrument) may be populated +with codes from **different ontologies** depending on the record — and +each ontology should map to a different Phenopacket block. + +The canonical example is a "specific finding" field that stores either an +HPO term (a phenotypic abnormality) or a MONDO term (a disease entity). +Both are clinically valid answers to the same question, but they belong in +different Phenopacket blocks: +.. list-table:: + :widths: 20 20 60 + :header-rows: 1 + + * - Ontology prefix + - Phenopacket block + - Semantic basis + * - ``HP:`` + - ``phenotypicFeatures`` + - HPO terms describe phenotypic abnormalities + (``PhenotypicFeature.type``) + * - ``MONDO:`` + - ``diseases`` + - MONDO terms describe disease entities (``Disease.term``) + * - ``OMIM:`` + - ``diseases`` + - OMIM identifiers describe disease entities + * - ``ORDO:`` + - ``diseases`` + - Orphanet identifiers describe rare disease entities + +These defaults are grounded in the GA4GH Phenopacket v2 schema and are +always active. They can be extended or overridden via the ``rules`` key +(see below). + +**Configuration** + +Add an ``ontology_routing`` key to your ``mapping_configs``:: + + mapping_configs = { + ... + "ontology_routing": { + "enabled": True, + + # Instruments whose repeated elements should be inspected + "instruments": [ + "your_mixed_instrument", + "another_mixed_instrument", + ], + + # Per-instrument: which fields to scan for routable codes. + # These are the type_field_1…N names from your mapping block. + # If omitted, all string fields are scanned (slower, zero-config). + "scan_fields": { + "your_mixed_instrument": [ + "field_holding_hp_or_mondo_1", + "field_holding_hp_or_mondo_2", + ], + "another_mixed_instrument": [ + "field_a", + "field_b", + ], + }, + + # Per-instrument: onset date fields. + # The first non-empty value is used as Disease.onset when a + # MONDO-coded element is routed to the diseases block. + "onset_fields": { + "your_mixed_instrument": [ + "onset_date_1", + "onset_date_2", + ], + "another_mixed_instrument": [ + "condition_onset_date", + ], + }, + + # Optional: override or extend the built-in prefix→block rules. + # Useful if your data model uses a non-standard ontology. + # "rules": { + # "HP": "phenotypicFeatures", + # "MONDO": "diseases", + # "MYCUSTOM": "diseases", + # } + }, + ... + } + +When ``ontology_routing`` is absent from ``mapping_configs``, the adapter +is never called and pipeline behaviour is identical to before. + +**What it does** + +Before any mapper runs, the adapter: + +1. Iterates over ``data["repeated_elements"]``. +2. For each element from a configured instrument, scans the configured + (or all) string fields for a value whose ontology prefix is in the + routing rules. +3. Routes the element: + + - **HP-coded** → placed in ``data["__routed__phenotypicFeatures"]`` + *unchanged*. The ``PhenotypicFeatureMapper`` processes it normally, + including multi-onset if configured. + - **MONDO/OMIM/ORDO-coded** → *normalized* to the ``term_field_1`` / + ``onset_date_field`` convention and placed in + ``data["__routed__diseases"]``. The ``DiseaseMapper`` consumes it + via the same path it uses for all other disease data. + +4. Elements with no routable code (e.g. a SNOMED sub-field that holds only + SNOMED values) are left in the original ``repeated_elements`` stream and + processed normally by whatever mapper is configured for that instrument. + +The original ``repeated_elements`` list is **never mutated**. + +**Data shape expected** + +A typical mixed element:: + + { + "redcap_repeat_instrument": "infections_initial_form", + "redcap_repeat_instance": 2, + "infections_initial_form": { + "type_of_infection": "snomedct_127856007", ← SNOMED category, ignored + "snomedct_127856007": "mondo_0043653", ← MONDO → diseases + "infection_severity": "hp_0012826", + "infection_date": "2023-02-01" + } + } + +The adapter detects ``mondo_0043653`` in ``snomedct_127856007``, routes +the element to ``diseases``, and normalizes it to:: + + { + "term_field_1": "mondo_0043653", + "onset_date_field": "2023-02-01", + "onset_category_field": None, + "excluded_field": None, + "primary_site_field": None, + "__source_instrument": "infections_initial_form", + "__source_instance": 2 + } + +**Validation integration** + +The adapter also provides :func:`check_prefix_placement`, which is called +automatically by ``validate_phenopackets`` after each file is written. It +inspects the serialized Phenopacket JSON and emits **non-fatal soft warnings** +when: + +- An HP: term appears in ``diseases`` (likely a routing error) +- A non-HP: term appears in ``phenotypicFeatures`` (MONDO/OMIM in a + features block) + +These warnings appear in the validation output but do not cause the +phenopacket to fail validation. They are intended to help data curators +catch misconfigured routing rules. + +**Adding the adapter to your mapping config** + +If you are building a custom data model and have instruments with mixed +ontology codes, add the ``ontology_routing`` block alongside your existing +``phenotypicFeatures`` and ``diseases`` configurations. No changes to +the mapper classes or the rest of the pipeline are needed. + +.. tip:: + Combine this adapter with the :ref:`multi-onset-adapter` when MONDO-routed + elements are diseases observed multiple times. Configure ``multi_onset`` + in the base ``phenotypicFeatures`` block; the routing adapter handles the + HP/MONDO split first, and the multi-onset adapter then expands the HP + elements per date. + _____________________________________________________________________________________ .. _troubleshooting: From 05b470641e250bba66f3d6ac4169b83015b48d67 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:50:00 +0200 Subject: [PATCH 10/28] updated index figure subheading --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8856c9c35..cd2970564 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,14 +22,14 @@ ________________________________________________________________________________ .. figure:: _static/res/rarelink_overview.png Overview of the RareLink framework integrated with a local REDCap instance. - The framework comprises the RareLink instruments based on the `RD-CDM `_, + The framework comprises the REDCap instruments based on the `RD-CDM `_, the RareLink API, the toFHIR & CDIS module for export and import to HL7 FHIR IPS and Genomic Reporting Resources, and the RareLink-Phenopackets module for direct export to GA4GH Phenopackets. Additionally, the LinkML-based import mapper facilitates data import from tabular databases into REDCap, while the Manual Data Capture Guide assists with the manual data capture of - RD-CDM data in a local REDCap project. Click `here `_ - to see the figure in full size. + RD-CDM data in a local REDCap project. The framework is also configurable + for disease-specific extensions or custom data models. ________________________________________________________________________________ From 81cb118e142ad4dc8f5153bd49a28437b6ed24db Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:50:25 +0200 Subject: [PATCH 11/28] added ontology_routing adapter --- src/rarelink/phenopackets/adapter/__init__.py | 13 +- .../phenopackets/adapter/multi_onset.py | 16 -- .../adapter/ontology_routing_adapter.py | 261 ++++++++++++++++++ 3 files changed, 272 insertions(+), 18 deletions(-) create mode 100644 src/rarelink/phenopackets/adapter/ontology_routing_adapter.py diff --git a/src/rarelink/phenopackets/adapter/__init__.py b/src/rarelink/phenopackets/adapter/__init__.py index 6f025b71c..c9e8a81d8 100644 --- a/src/rarelink/phenopackets/adapter/__init__.py +++ b/src/rarelink/phenopackets/adapter/__init__.py @@ -4,8 +4,17 @@ """ from .multi_onset import multi_onset_adapter +from .ontology_routing_adapter import ( + apply_ontology_routing, + get_routed_phenotypic_elements, + get_routed_disease_dicts, + check_prefix_placement, +) __all__ = [ - "multi_onset_adapter" - + "multi_onset_adapter", + "apply_ontology_routing", + "get_routed_phenotypic_elements", + "get_routed_disease_dicts", + "check_prefix_placement" ] \ No newline at end of file diff --git a/src/rarelink/phenopackets/adapter/multi_onset.py b/src/rarelink/phenopackets/adapter/multi_onset.py index eb894a04f..6df58eca7 100644 --- a/src/rarelink/phenopackets/adapter/multi_onset.py +++ b/src/rarelink/phenopackets/adapter/multi_onset.py @@ -27,55 +27,40 @@ def multi_onset_adapter( Returns: List[Any]: A list of feature blocks. """ - # Create the base feature using the provided mapping function base_feature = mapping_func(feature_type, feature_data, processor, dob) - # If multi_onset is not enabled or no onset_date_fields defined, return the base feature. if not processor.mapping_config.get("multi_onset", False): logger.debug("Multi-onset not enabled, returning single feature") return [base_feature] - # Get onset fields from config onset_fields = processor.mapping_config.get("onset_date_fields", []) if not onset_fields: logger.debug("No onset date fields configured, returning single feature") return [base_feature] - # Create a list to store all features features = [] found_onset = False - # Process each onset field for field in onset_fields: - # Strip any instrument prefix from the field name field_name = field.split(".")[-1] if "." in field else field - # Try to get the onset value onset_value = feature_data.get(field_name) if onset_value: found_onset = True try: - # Convert the onset value to a string if needed onset_date_str = onset_value if isinstance(onset_value, str) else str(onset_value) - # Ensure dob is a string in proper format dob_str = dob if isinstance(dob, str) else (str(dob) if dob else None) logger.debug(f"Processing onset field '{field_name}' with value '{onset_date_str}'") - # Calculate age at onset iso_age = processor.convert_date_to_iso_age(onset_date_str, dob_str) if iso_age: - # Create a new onset TimeElement onset = TimeElement(age=Age(iso8601duration=iso_age)) - # Create a deep copy of the base feature feature_copy = copy.deepcopy(base_feature) - # Clear the existing onset field and set the new one feature_copy.ClearField("onset") feature_copy.onset.MergeFrom(onset) - - # Add the feature to our list features.append(feature_copy) logger.debug(f"Created feature with onset from field '{field_name}': {iso_age}") except Exception as e: @@ -83,7 +68,6 @@ def multi_onset_adapter( import traceback logger.debug(traceback.format_exc()) - # If no valid onset values were found, return the base feature if not found_onset or not features: logger.debug("No valid onset values found, returning base feature") return [base_feature] diff --git a/src/rarelink/phenopackets/adapter/ontology_routing_adapter.py b/src/rarelink/phenopackets/adapter/ontology_routing_adapter.py new file mode 100644 index 000000000..3a900b85d --- /dev/null +++ b/src/rarelink/phenopackets/adapter/ontology_routing_adapter.py @@ -0,0 +1,261 @@ +import logging +import re +from typing import Any, Dict, List, Optional, Tuple +from rarelink.utils.code_processing import process_code + +logger = logging.getLogger(__name__) + +# Semantic defaults based on the Phenopacket Schema v2.0 +DEFAULT_ROUTING_RULES: Dict[str, str] = { + "HP": "phenotypicFeatures", + "MONDO": "diseases", + "OMIM": "diseases", + "ORDO": "diseases", +} + +_PREFIX_RE = re.compile(r"^([A-Za-z][A-Za-z0-9]*)[:_]") + +_SKIP_FIELDS = frozenset({ + "redcap_repeat_instrument", + "redcap_repeat_instance", + "type_of_infection", + "type_of_condition", +}) + +# Public API +def should_route(mapping_configs: Dict[str, Any]) -> bool: + """Return True when ontology routing is configured and enabled.""" + cfg = mapping_configs.get("ontology_routing", {}) + return bool(cfg) and cfg.get("enabled", True) + + +def apply_ontology_routing( + data: Dict[str, Any], + mapping_configs: Dict[str, Any], +) -> Dict[str, Any]: + """ + Inspect ``data["repeated_elements"]``, route each element from a + configured instrument to either ``phenotypicFeatures`` or ``diseases`` + based on the ontology prefix of its type-field value. + + Returns a shallow copy of ``data`` with two extra keys: + + * ``__routed__phenotypicFeatures`` — HP-bearing elements (unchanged dicts) + * ``__routed__diseases`` — MONDO/OMIM/ORDO elements normalized + to the ``term_field_1`` / ``onset_date_field`` convention + + The original ``repeated_elements`` list is never mutated. + """ + routing_cfg = mapping_configs.get("ontology_routing", {}) + rules = _build_rules(routing_cfg) + instruments = set(routing_cfg.get("instruments", [])) + scan_fields_map: Dict[str, List[str]] = routing_cfg.get("scan_fields", {}) + onset_fields_map: Dict[str, List[str]] = routing_cfg.get("onset_fields", {}) + + routed_pf: List[Dict[str, Any]] = [] + routed_diseases: List[Dict[str, Any]] = [] + skipped = 0 + + for element in data.get("repeated_elements", []): + inst = element.get("redcap_repeat_instrument", "") + if inst not in instruments: + continue + + inner = element.get(inst, {}) + if not inner: + continue + + scan = scan_fields_map.get(inst) # None means scan all fields + code, field = _find_routable_code(inner, scan) + + if code is None: + skipped += 1 + logger.debug( + f"[ontology_routing] {inst} #{element.get('redcap_repeat_instance')}" + f": no routable code found — element left in original stream" + ) + continue + + prefix = _get_prefix(code) + destination = rules.get(prefix.upper()) if prefix else None + + if destination is None: + logger.debug( + f"[ontology_routing] {inst} #{element.get('redcap_repeat_instance')}" + f": prefix '{prefix}' not in routing rules — skipping" + ) + skipped += 1 + continue + + if destination == "phenotypicFeatures": + routed_pf.append(element) + logger.debug( + f"[ontology_routing] {inst} #{element.get('redcap_repeat_instance')}" + f": {code} ({prefix}) → phenotypicFeatures" + ) + + elif destination == "diseases": + onset_fields = onset_fields_map.get(inst, []) + normalized = _normalize_for_disease_mapper( + inner, code, onset_fields, + instance=element.get("redcap_repeat_instance"), + instrument=inst, + ) + routed_diseases.append(normalized) + logger.debug( + f"[ontology_routing] {inst} #{element.get('redcap_repeat_instance')}" + f": {code} ({prefix}) → diseases (normalized onset=" + f"{normalized.get('onset_date_field')})" + ) + + logger.debug( + f"[ontology_routing] result: " + f"{len(routed_pf)} → phenotypicFeatures, " + f"{len(routed_diseases)} → diseases, " + f"{skipped} skipped" + ) + + augmented = dict(data) + augmented["__routed__phenotypicFeatures"] = routed_pf + augmented["__routed__diseases"] = routed_diseases + return augmented + + +def get_routed_phenotypic_elements(data: Dict[str, Any]) -> List[Dict[str, Any]]: + """Return pre-routed HP elements, or empty list if routing was not applied.""" + return data.get("__routed__phenotypicFeatures", []) + + +def get_routed_disease_dicts(data: Dict[str, Any]) -> List[Dict[str, Any]]: + """Return pre-routed normalized disease dicts, or empty list.""" + return data.get("__routed__diseases", []) + + +# --------------------------------------------------------------------------- +# Validation helper — for use in validate.py soft-warning checks +# --------------------------------------------------------------------------- + +def check_prefix_placement(phenopacket_json: Dict[str, Any]) -> List[str]: + """ + Inspect a serialized Phenopacket JSON dict and return a list of + soft-warning messages where ontology terms appear in semantically + wrong blocks (e.g. an HP: term in ``diseases``). + + These are warnings only — a phenopacket can still be structurally + valid while violating these semantic conventions. They are surfaced + in the validation output to help data curators catch routing errors. + """ + warnings: List[str] = [] + + # HP codes must only appear as PhenotypicFeature.type + for i, pf in enumerate(phenopacket_json.get("phenotypicFeatures", [])): + term_id = pf.get("type", {}).get("id", "") + if term_id and not term_id.startswith("HP:"): + warnings.append( + f"phenotypicFeatures[{i}].type.id = '{term_id}' — " + f"expected HP: prefix (GA4GH Phenopacket v2 convention)" + ) + + # MONDO/OMIM/ORDO codes must only appear as Disease.term + disease_prefixes = {"HP"} + for i, disease in enumerate(phenopacket_json.get("diseases", [])): + term_id = disease.get("term", {}).get("id", "") + if term_id: + prefix = term_id.split(":")[0].upper() if ":" in term_id else "" + if prefix in disease_prefixes: + warnings.append( + f"diseases[{i}].term.id = '{term_id}' — " + f"HP: terms describe phenotypic features, not diseases " + f"(GA4GH Phenopacket v2 convention)" + ) + + return warnings + + + +# Internal helpers ----------------------------------------------------------- +def _build_rules(routing_cfg: Dict[str, Any]) -> Dict[str, str]: + """Merge user-supplied rules over the semantic defaults.""" + rules = dict(DEFAULT_ROUTING_RULES) + user_rules = routing_cfg.get("rules", {}) + rules.update({k.upper(): v for k, v in user_rules.items()}) + return rules + + +def _get_prefix(code: str) -> Optional[str]: + if not code: + return None + try: + normalised = process_code(code) + if ":" in normalised: + return normalised.split(":")[0].upper() + except Exception: + pass + return None + + +def _find_routable_code( + inner: Dict[str, Any], + scan_fields: Optional[List[str]], +) -> Tuple[Optional[str], Optional[str]]: + """ + Scan ``inner`` for the first non-empty string value whose normalised + prefix appears in ``DEFAULT_ROUTING_RULES``. + + Args: + inner: The instrument-level dict from a repeated element. + scan_fields: Explicit list of field names to check, or ``None`` + to scan all non-structural fields. + + Returns: + ``(code_value, field_name)`` or ``(None, None)``. + """ + fields_to_check = scan_fields if scan_fields else [ + k for k in inner if k not in _SKIP_FIELDS + ] + + for field in fields_to_check: + value = inner.get(field) + if not value or not isinstance(value, str): + continue + prefix = _get_prefix(value) + if prefix and prefix.upper() in DEFAULT_ROUTING_RULES: + return value, field + + return None, None + + +def _normalize_for_disease_mapper( + inner: Dict[str, Any], + code: str, + onset_fields: List[str], + instance: Optional[int] = None, + instrument: Optional[str] = None, +) -> Dict[str, Any]: + """ + Produce a normalized dict in the ``term_field_1`` / ``onset_date_field`` + convention expected by DiseaseMapper. + + The ``code`` value (e.g. ``"mondo_0043653"``) is placed in + ``term_field_1``. The first non-empty value from ``onset_fields`` + is placed in ``onset_date_field``. + + ``__source_instrument`` and ``__source_instance`` are added for + traceability and are ignored by all mappers. + """ + onset_date: Optional[str] = None + for field in onset_fields: + val = inner.get(field) + if val and isinstance(val, str): + onset_date = val + break + + return { + "term_field_1": code, + "onset_date_field": onset_date, + "onset_category_field": None, + "excluded_field": None, + "primary_site_field": None, + "__source_instrument": instrument, + "__source_instance": instance, + } \ No newline at end of file From 41ef91f03728a5c811679da7eb49f1909e8ad326 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:50:44 +0200 Subject: [PATCH 12/28] test for adapter --- .../adapter/test_ontology_routing_adapter.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/phenopackets/adapter/test_ontology_routing_adapter.py diff --git a/tests/phenopackets/adapter/test_ontology_routing_adapter.py b/tests/phenopackets/adapter/test_ontology_routing_adapter.py new file mode 100644 index 000000000..acc957ce7 --- /dev/null +++ b/tests/phenopackets/adapter/test_ontology_routing_adapter.py @@ -0,0 +1,115 @@ +# tests/phenopackets/adapter/test_ontology_routing_adapter.py +""" +Tests for the ontology routing adapter. + +Three tests cover the essential behaviour: + 1. HP-coded elements route to phenotypicFeatures (unchanged) + 2. MONDO-coded elements route to diseases (normalized) + 3. Elements with no routable code are skipped without error + +Uses the exact data shape from the CIEINR test fixture. +""" + +from rarelink.phenopackets.adapter.ontology_routing_adapter import ( + apply_ontology_routing, + get_routed_disease_dicts, + get_routed_phenotypic_elements, +) + +# --------------------------------------------------------------------------- +# Shared config and fixture +# --------------------------------------------------------------------------- + +ROUTING_CONFIG = { + "ontology_routing": { + "enabled": True, + "instruments": ["infections_initial_form"], + "scan_fields": { + "infections_initial_form": [ + "snomedct_21483005", # CNS → HP codes + "snomedct_127856007", # Skin → HP or MONDO + "snomedct_20139000", # Respiratory → HP codes + ], + }, + "onset_fields": { + "infections_initial_form": ["infection_date", "infection_date_2"], + }, + } +} + +# Three infections elements: +# #1 — HP code → phenotypicFeatures +# #2 — MONDO code → diseases +# #3 — SNOMED value (no routable prefix) → skipped +RECORD = { + "record_id": "test_1", + "repeated_elements": [ + { + "redcap_repeat_instrument": "infections_initial_form", + "redcap_repeat_instance": 1, + "infections_initial_form": { + "type_of_infection": "snomedct_21483005", + "snomedct_21483005": "hp_0002383", # HP → phenotypicFeatures + "infection_severity": "hp_0012826", + "infection_date": "2022-02-01", + }, + }, + { + "redcap_repeat_instrument": "infections_initial_form", + "redcap_repeat_instance": 2, + "infections_initial_form": { + "type_of_infection": "snomedct_127856007", + "snomedct_127856007": "mondo_0043653", # MONDO → diseases + "infection_severity": "hp_0012826", + "infection_date": "2023-02-01", + }, + }, + { + "redcap_repeat_instrument": "infections_initial_form", + "redcap_repeat_instance": 3, + "infections_initial_form": { + "type_of_infection": "snomedct_20139000", + "snomedct_20139000": "snomedct_233604007", # SNOMED → skip + "infection_date": "2024-01-01", + }, + }, + ], +} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +def test_hp_element_routes_to_phenotypic_features(): + """HP-coded element ends up in phenotypicFeatures, unchanged.""" + data = apply_ontology_routing(RECORD, ROUTING_CONFIG) + pf = get_routed_phenotypic_elements(data) + + assert len(pf) == 1 + assert pf[0]["redcap_repeat_instance"] == 1 + assert pf[0]["infections_initial_form"]["snomedct_21483005"] == "hp_0002383" + + +def test_mondo_element_routes_to_diseases_normalized(): + """MONDO-coded element ends up in diseases, normalized to term_field_1 convention.""" + data = apply_ontology_routing(RECORD, ROUTING_CONFIG) + diseases = get_routed_disease_dicts(data) + + assert len(diseases) == 1 + d = diseases[0] + assert d["term_field_1"] == "mondo_0043653" + assert d["onset_date_field"] == "2023-02-01" + assert d["__source_instance"] == 2 + + +def test_element_with_no_routable_code_is_skipped(): + """Element with an unrecognised prefix is not routed and original data is untouched.""" + data = apply_ontology_routing(RECORD, ROUTING_CONFIG) + + # Only HP (instance 1) and MONDO (instance 2) were routed + assert len(get_routed_phenotypic_elements(data)) == 1 + assert len(get_routed_disease_dicts(data)) == 1 + + # Original list untouched — no mutation + assert len(data["repeated_elements"]) == 3 \ No newline at end of file From 97e863e61c6c407ac1b655b917220e26a8e1570f Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:51:31 +0200 Subject: [PATCH 13/28] updated phenopackets export adn validation with new functions and cli overview --- src/rarelink/cli/phenopackets/export.py | 101 +++++- src/rarelink/phenopackets/create.py | 420 +++++++++++++++++------- src/rarelink/phenopackets/validate.py | 160 ++++----- 3 files changed, 472 insertions(+), 209 deletions(-) diff --git a/src/rarelink/cli/phenopackets/export.py b/src/rarelink/cli/phenopackets/export.py index 4d3322e89..5f99e028a 100644 --- a/src/rarelink/cli/phenopackets/export.py +++ b/src/rarelink/cli/phenopackets/export.py @@ -282,6 +282,17 @@ def fetch_label(code: str, enum_class=None, label_dict=None): from rarelink.phenopackets.pipeline import phenopacket_pipeline # ── Step 7a: Phase 1 progress bar — Creating ───────────────────────────── + # Mute all logging output while the progress bars are live. + # basicConfig attaches handlers to the ROOT logger, so setting the level + # on a child namespace ("rarelink") has no effect — the root handlers still + # fire. Instead we temporarily remove all root handlers and restore them + # after both bars finish. In --debug mode logging stays active. + import logging as _logging + _root_logger = _logging.getLogger() + _saved_handlers = _root_logger.handlers[:] + if not debug: + _root_logger.handlers = [] + create_progress, create_task = _make_progress( "Creating phenopackets", total ) @@ -298,9 +309,10 @@ def on_created(record_id, success, error): timeout=timeout, debug=debug, progress_callback=on_created, - validation_callback=None, # not wired yet — phase 2 below + validation_callback=None, ) except Exception as e: + _root_logger.handlers = _saved_handlers # restore before printing typer.secho( error_text(f"❌ Pipeline failed: {str(e)}"), fg=typer.colors.RED, @@ -311,6 +323,11 @@ def on_created(record_id, success, error): raise typer.Exit(1) # ── Step 7b: Phase 2 progress bar — Validating ─────────────────────────── + # Collect prefix-placement warnings from the detail strings returned by + # validate_phenopackets. They are non-fatal (success=True) but contain + # "Ontology prefix warnings" when a MONDO/HP term is in the wrong block. + _prefix_warnings: list = [] + n_to_validate = result.n_created if n_to_validate > 0: validate_progress, validate_task = _make_progress( @@ -319,6 +336,13 @@ def on_created(record_id, success, error): with validate_progress: def on_validated(file_path, success, error): validate_progress.advance(validate_task) + # Harvest prefix warnings from the detail string even on success + if error and "Ontology prefix warnings" in error: + fname = Path(file_path).name + for line in error.splitlines(): + line = line.strip() + if line.startswith("⚠"): + _prefix_warnings.append(f"{fname}: {line.lstrip('⚠').strip()}") _run_write_and_validate( phenopackets=result.phenopackets, @@ -328,6 +352,9 @@ def on_validated(file_path, success, error): debug=debug, ) + # Restore root logger handlers — logging resumes for the summary section + _root_logger.handlers = _saved_handlers + # ── Step 8: Summary ────────────────────────────────────────────────────── console.print() _print_summary( @@ -339,6 +366,7 @@ def on_validated(file_path, success, error): output_dir=output_dir, failed_creations=result.failed_creations, failed_validations=result.failed_validations, + prefix_warnings=_prefix_warnings, ) end_of_section_separator() @@ -385,7 +413,8 @@ def _run_write_and_validate( ok, details = validate_phenopackets(file_path) if ok: if validation_callback: - validation_callback(str(file_path), success=True, error=None) + # Pass details even on success — may contain prefix warnings + validation_callback(str(file_path), success=True, error=details) else: result.failed_validations.append( {"file": str(file_path), "error": details} @@ -410,10 +439,13 @@ def _print_summary( output_dir: Path, failed_creations: list, failed_validations: list, + prefix_warnings: list = None, ): - """Render a clean Rich summary table with optional failure details.""" + """Render a clean Rich summary table with failure details and prefix warnings.""" + prefix_warnings = prefix_warnings or [] all_ok = (n_failed_creation == 0 and n_failed_validation == 0) + # ── Stats table ────────────────────────────────────────────────────────── table = Table( title="Export Summary", show_header=True, @@ -425,26 +457,47 @@ def _print_summary( table.add_column("Stage", style="dim", width=24) table.add_column("Passed", justify="right", style="green") table.add_column("Failed", justify="right") + table.add_column("Warnings", justify="right") table.add_column("Total", justify="right", style="bold") def _fail_style(n: int) -> Text: return Text(str(n), style="bold red" if n > 0 else "green") - table.add_row("Creation", str(n_created), _fail_style(n_failed_creation), str(total)) - table.add_row("Validation", str(n_validated), _fail_style(n_failed_validation), str(n_created)) + def _warn_style(n: int) -> Text: + return Text(str(n), style="bold yellow" if n > 0 else "dim") + + table.add_row( + "Creation", + str(n_created), + _fail_style(n_failed_creation), + _warn_style(0), + str(total), + ) + table.add_row( + "Validation", + str(n_validated), + _fail_style(n_failed_validation), + _warn_style(len(prefix_warnings)), + str(n_created), + ) console.print(table) console.print(f" 📂 Output directory: [bold]{output_dir}[/bold]\n") - if all_ok: + # ── Success / failure panel ─────────────────────────────────────────────── + if all_ok and not prefix_warnings: console.print( - Panel( - "✅ [bold green]All phenopackets created and validated successfully![/bold green]", - border_style="green", + "✅ [green]All phenopackets created and validated successfully![/green]", ) - ) return + if all_ok and prefix_warnings: + console.print( + "✅ [green]All phenopackets created and validated successfully.[/green]\n" + "[yellow]Ontology prefix warnings were found — see below.[/yellow]", + ) + + # ── Creation failures ───────────────────────────────────────────────────── if failed_creations: console.print() fail_table = Table( @@ -458,6 +511,7 @@ def _fail_style(n: int) -> Text: fail_table.add_row(str(f["record_id"]), f["error"]) console.print(fail_table) + # ── Validation failures & warnings ─────────────────────────────────────── if failed_validations: console.print() val_table = Table( @@ -471,6 +525,33 @@ def _fail_style(n: int) -> Text: val_table.add_row(Path(f["file"]).name, f["error"]) console.print(val_table) + + + + # ── Failures (inline, not tables) ──────────────────────────────────────── + if failed_creations: + console.print(f"\n [red]❌ Creation failures ({len(failed_creations)}):[/red]") + for f in failed_creations[:5]: # cap at 5 inline; rest in failures.json + console.print(f" [dim]{f['record_id']}[/dim] {f['error']}") + if len(failed_creations) > 5: + console.print(f" [dim]... and {len(failed_creations) - 5} more — see failures.json[/dim]") + + if failed_validations: + console.print(f"\n [yellow]⚠ Validation failures ({len(failed_validations)}):[/yellow]") + for f in failed_validations[:5]: + console.print(f" [dim]{Path(f['file']).name}[/dim] {f['error'][:80]}") + if len(failed_validations) > 5: + console.print(f" [dim]... and {len(failed_validations) - 5} more — see failures.json[/dim]") + + # ── Prefix warnings (inline) ────────────────────────────────────────────── + if prefix_warnings: + console.print(f"\n [yellow]⚠ Ontology prefix warnings ({len(prefix_warnings)}) — non-fatal:[/yellow]") + for w in prefix_warnings[:5]: + console.print(f" [dim]{w}[/dim]") + if len(prefix_warnings) > 5: + console.print(f" [dim]... and {len(prefix_warnings) - 5} more[/dim]") + + failure_file = output_dir / "failures.json" if failure_file.exists(): console.print(f"\n 💾 Full failure report: [bold]{failure_file}[/bold]") diff --git a/src/rarelink/phenopackets/create.py b/src/rarelink/phenopackets/create.py index d41f92701..ef8593376 100644 --- a/src/rarelink/phenopackets/create.py +++ b/src/rarelink/phenopackets/create.py @@ -1,4 +1,3 @@ -# src/rarelink/phenopackets/create.py from phenopackets import Phenopacket import logging import traceback @@ -8,7 +7,6 @@ from rarelink.rarelink_cdm import get_codesystems_container_class from rarelink.phenopackets.mappings.metadata_mapper import collect_used_prefixes_from_blocks -# Import new mapper classes from rarelink.phenopackets.mappings.individual_mapper import IndividualMapper from rarelink.phenopackets.mappings.vital_status_mapper import VitalStatusMapper from rarelink.phenopackets.mappings.phenotypic_feature_mapper import PhenotypicFeatureMapper @@ -18,33 +16,37 @@ from rarelink.phenopackets.mappings.variation_descriptor_mapper import VariationDescriptorMapper from rarelink.phenopackets.mappings.interpretation_mapper import InterpretationMapper from rarelink.phenopackets.mappings.metadata_mapper import MetadataMapper - +from rarelink.phenopackets.adapter.ontology_routing_adapter import ( + should_route, + apply_ontology_routing, + get_routed_phenotypic_elements, + get_routed_disease_dicts, +) logger = logging.getLogger(__name__) def create_phenopacket( - data: dict, - created_by: str, + data: dict, + created_by: str, mapping_configs: Optional[Dict[str, Any]] = None, debug: bool = False ) -> Phenopacket: """ - Creates a Phenopacket for an individual record with flexible mapping configurations. - Refactored to use mapper classes for improved organization. - + Creates a Phenopacket for an individual record with flexible mapping + configurations. + Args: - data (dict): Input data. - created_by (str): Creator's name. - mapping_configs (dict, optional): Mapping configurations for different blocks. - debug (bool): Enable debug mode. - + data (dict): Input record data. + created_by (str): Creator's name (written into MetaData). + mapping_configs (dict): Mapping configurations for all blocks. + debug (bool): Enable verbose debug logging. + Returns: - Phenopacket: The constructed Phenopacket. + Phenopacket: The fully constructed Phenopacket. """ if not mapping_configs: raise ValueError("Mapping configurations are required.") - # Set logging level logging_level = logging.DEBUG if debug else logging.INFO logger.setLevel(logging_level) @@ -53,45 +55,54 @@ def create_phenopacket( if debug: logger.debug(f"Processing record ID: {record_id}") - # --- Helper: Create a processor by merging outer keys --- + # Ontology routing (opt-in) + if should_route(mapping_configs): + data = apply_ontology_routing(data, mapping_configs) + logger.debug( + f"[routing] phenotypicFeatures: " + f"{len(get_routed_phenotypic_elements(data))}, " + f"diseases: {len(get_routed_disease_dicts(data))}" + ) + + # ── Helper: build a DataProcessor from a named config block ────────── def create_processor(block: str, required: bool = False): - """ - Create a DataProcessor for a given block by merging the outer configuration keys - into the mapping config. If the block configuration is a list, use its first element. - Also, if 'instrument_name' is a collection, convert it to a list of strings and - set 'redcap_repeat_instrument' to the first element. - """ config = mapping_configs.get(block, {}) - # If the configuration is a list, use the first element as the base config. if isinstance(config, list): config = config[0] if required and not config: - raise ValueError(f"Required mapping configuration '{block}' missing.") - # Start with a copy of the mapping block + raise ValueError( + f"Required mapping configuration '{block}' missing." + ) mapping_block = config.get("mapping_block", {}).copy() - # Merge other keys from the outer config for key, value in config.items(): if key != "mapping_block": mapping_block[key] = value - # Convert instrument_name to a list of strings if needed. if "instrument_name" in mapping_block: inst = mapping_block["instrument_name"] if not isinstance(inst, str): - # Convert set or list items to strings. - inst_list = [str(x) for x in inst] if isinstance(inst, (list, set)) else [str(inst)] + inst_list = ( + [str(x) for x in inst] + if isinstance(inst, (list, set)) + else [str(inst)] + ) mapping_block["instrument_name"] = inst_list else: - mapping_block["instrument_name"] = [inst] # wrap a single string in a list - # Ensure that "redcap_repeat_instrument" is set to the first instrument. - if "instrument_name" in mapping_block and "redcap_repeat_instrument" not in mapping_block: - mapping_block["redcap_repeat_instrument"] = mapping_block["instrument_name"][0] + mapping_block["instrument_name"] = [inst] + if ( + "instrument_name" in mapping_block + and "redcap_repeat_instrument" not in mapping_block + ): + mapping_block["redcap_repeat_instrument"] = ( + mapping_block["instrument_name"][0] + ) processor = DataProcessor(mapping_config=mapping_block) processor.enable_debug(debug) - add_enum_classes_to_processor(processor, config.get("enum_classes", {})) + add_enum_classes_to_processor( + processor, config.get("enum_classes", {}) + ) return processor, config - - # --- Individual & Vital Status --- + # ── Individual & Vital Status ──────────────────────────────────────── individual_processor, _ = create_processor("individual", required=True) try: dob_field = individual_processor.get_field(data, "date_of_birth_field") @@ -109,176 +120,254 @@ def create_processor(block: str, required: bool = False): individual_mapper = IndividualMapper(individual_processor) individual = individual_mapper.map(data, vital_status=vital_status) - # --- Phenotypic Features --- + # ── Phenotypic Features ────────────────────────────────────────────── phenotypic_features = [] phenotypic_config = mapping_configs.get("phenotypicFeatures") - - # Ensure we store the full data context for proper modifier scoping - full_data = data - + full_data = data + + # if ontology_routing is active + routed_pf = get_routed_phenotypic_elements(data) + if routed_pf: + base_pf_config = ( + phenotypic_config[0] + if isinstance(phenotypic_config, list) and phenotypic_config + else phenotypic_config or {} + ) + routed_pf_config = _build_routed_pf_config(base_pf_config, routed_pf) + + if isinstance(phenotypic_config, list): + phenotypic_config = list(phenotypic_config) + [routed_pf_config] + elif phenotypic_config: + phenotypic_config = [phenotypic_config, routed_pf_config] + else: + phenotypic_config = [routed_pf_config] + if isinstance(phenotypic_config, list): - logger.debug(f"Processing {len(phenotypic_config)} phenotypic feature configurations") + logger.debug( + f"Processing {len(phenotypic_config)} phenotypic feature " + f"configurations" + ) for i, config in enumerate(phenotypic_config): try: - # Start with base mapping block - proc = DataProcessor(mapping_config=config.get("mapping_block", {}).copy()) - - # Set full_data for proper feature & modifier scoping - proc.mapping_config['full_data'] = full_data - - # Merge outer keys + proc = DataProcessor( + mapping_config=config.get("mapping_block", {}).copy() + ) + proc.mapping_config["full_data"] = full_data for key, value in config.items(): if key != "mapping_block": proc.mapping_config[key] = value - - # Ensure repeat instrument is set - if "instrument_name" in proc.mapping_config and "redcap_repeat_instrument" not in proc.mapping_config: - proc.mapping_config["redcap_repeat_instrument"] = proc.mapping_config["instrument_name"] - + if ( + "instrument_name" in proc.mapping_config + and "redcap_repeat_instrument" + not in proc.mapping_config + ): + proc.mapping_config["redcap_repeat_instrument"] = ( + proc.mapping_config["instrument_name"] + ) proc.enable_debug(debug) - - # Add enum classes - add_enum_classes_to_processor(proc, config.get("enum_classes", {})) - - # Create mapper and map features + add_enum_classes_to_processor( + proc, config.get("enum_classes", {}) + ) feature_mapper = PhenotypicFeatureMapper(proc) - feats = feature_mapper.map(data, dob=individual.date_of_birth) - + feats = feature_mapper.map( + data, dob=individual.date_of_birth + ) if feats: - # Ensure each feature has appropriate modifiers only phenotypic_features.extend(feats) - logger.debug(f"Added {len(feats)} features from config {i+1}") + logger.debug( + f"Added {len(feats)} features from config {i + 1}" + ) except Exception as e: - logger.error(f"Error processing phenotypic feature config {i+1}: {e}") + logger.error( + f"Error processing phenotypic feature config " + f"{i + 1}: {e}" + ) if debug: logger.debug(traceback.format_exc()) else: - # Single configuration case proc, _ = create_processor("phenotypicFeatures") - - # Set full_data for proper feature & modifier scoping - proc.mapping_config['full_data'] = full_data - + proc.mapping_config["full_data"] = full_data feature_mapper = PhenotypicFeatureMapper(proc) - phenotypic_features = feature_mapper.map(data, dob=individual.date_of_birth) - + phenotypic_features = feature_mapper.map( + data, dob=individual.date_of_birth + ) + if debug: logger.debug(f"Total phenotypic features: {len(phenotypic_features)}") - - # Validate and deduplicate features if needed + + # Deduplicate features (guard against multi-config overlap) processed_features = [] feature_types_seen = set() - for feature in phenotypic_features: - # Skip invalid features if not feature or not feature.type or not feature.type.id: continue - - # For CIEINR/multi-instrument setups, deduplicate features by onset date - # This handles cases where the same feature appears in multiple configs feature_key = ( - feature.type.id, - str(getattr(feature.onset, 'age', None)) if hasattr(feature, 'onset') else None, - # Include a hash of modifiers if they exist - hash(tuple(sorted([m.id for m in feature.modifiers]))) if hasattr(feature, 'modifiers') and feature.modifiers else None + feature.type.id, + str(getattr(feature.onset, "age", None)) + if hasattr(feature, "onset") + else None, + hash(tuple(sorted([m.id for m in feature.modifiers]))) + if hasattr(feature, "modifiers") and feature.modifiers + else None, ) - if feature_key not in feature_types_seen: feature_types_seen.add(feature_key) processed_features.append(feature) - phenotypic_features = processed_features - - # --- Measurements --- + + # ── Measurements ───────────────────────────────────────────────────── measurements = [] measurement_config = mapping_configs.get("measurements") if isinstance(measurement_config, list): - logger.debug(f"Processing {len(measurement_config)} measurement configurations") + logger.debug( + f"Processing {len(measurement_config)} measurement configurations" + ) for i, config in enumerate(measurement_config): try: - proc = DataProcessor(mapping_config=config.get("mapping_block", {}).copy()) + proc = DataProcessor( + mapping_config=config.get("mapping_block", {}).copy() + ) for key, value in config.items(): if key != "mapping_block": proc.mapping_config[key] = value - if "instrument_name" in proc.mapping_config and "redcap_repeat_instrument" not in proc.mapping_config: - proc.mapping_config["redcap_repeat_instrument"] = proc.mapping_config["instrument_name"] + if ( + "instrument_name" in proc.mapping_config + and "redcap_repeat_instrument" + not in proc.mapping_config + ): + proc.mapping_config["redcap_repeat_instrument"] = ( + proc.mapping_config["instrument_name"] + ) proc.enable_debug(debug) - add_enum_classes_to_processor(proc, config.get("enum_classes", {})) + add_enum_classes_to_processor( + proc, config.get("enum_classes", {}) + ) measurement_mapper = MeasurementMapper(proc) - meas = measurement_mapper.map(data, dob=individual.date_of_birth) + meas = measurement_mapper.map( + data, dob=individual.date_of_birth + ) if meas: measurements.extend(meas) - logger.debug(f"Added {len(meas)} measurements from config {i+1}") + logger.debug( + f"Added {len(meas)} measurements from config {i + 1}" + ) except Exception as e: - logger.error(f"Error processing measurement config {i+1}: {e}") + logger.error( + f"Error processing measurement config {i + 1}: {e}" + ) if debug: logger.debug(traceback.format_exc()) else: proc, _ = create_processor("measurements") measurement_mapper = MeasurementMapper(proc) measurements = measurement_mapper.map( - data, dob=individual.date_of_birth) + data, dob=individual.date_of_birth + ) if debug: logger.debug(f"Total measurements: {len(measurements)}") - # --- Medical Actions (Procedures and Treatments) --- + # ── Medical Actions ─────────────────────────────────────────────────── medical_actions = [] proc_processor, _ = create_processor("medical_actions") medical_action_mapper = MedicalActionMapper(proc_processor) proc_actions = medical_action_mapper.map( - data, dob=individual.date_of_birth) + data, dob=individual.date_of_birth + ) if proc_actions: medical_actions.extend(proc_actions) logger.debug( - f"Added {len(proc_actions)} procedure-based medical actions") + f"Added {len(proc_actions)} procedure-based medical actions" + ) treatments_config = mapping_configs.get("treatments") if treatments_config: if isinstance(treatments_config, list): for i, config in enumerate(treatments_config): try: - # Use the helper to get a base processor from - # the first element of the treatments list. proc, _ = create_processor("treatments") - # Merge the specific treatment config overrides for key, value in config.items(): if key != "mapping_block": proc.mapping_config[key] = value proc.enable_debug(debug) add_enum_classes_to_processor( - proc, config.get("enum_classes", {})) + proc, config.get("enum_classes", {}) + ) treatment_mapper = MedicalActionMapper(proc) treat_actions = treatment_mapper.map( - data, dob=individual.date_of_birth) + data, dob=individual.date_of_birth + ) if treat_actions: medical_actions.extend(treat_actions) logger.debug( - f"Added {len(treat_actions)} " - f"treatment actions from config {i+1}") + f"Added {len(treat_actions)} treatment actions " + f"from config {i + 1}" + ) except Exception as e: logger.error( - f"Error processing treatment config {i+1}: {e}") + f"Error processing treatment config {i + 1}: {e}" + ) if debug: logger.debug(traceback.format_exc()) elif isinstance(treatments_config, dict): proc, _ = create_processor("treatments") treatment_mapper = MedicalActionMapper(proc) - treat_actions = treatment_mapper.map(data, dob=individual.date_of_birth) + treat_actions = treatment_mapper.map( + data, dob=individual.date_of_birth + ) if treat_actions: medical_actions.extend(treat_actions) - logger.debug(f"Added {len(treat_actions)} treatment actions") + logger.debug( + f"Added {len(treat_actions)} treatment actions" + ) if debug: logger.debug(f"Total medical actions: {len(medical_actions)}") - # --- Diseases --- + # ── Diseases ───────────────────────────────────────────────────────── disease_processor, _ = create_processor("diseases") disease_mapper = DiseaseMapper(disease_processor) diseases = disease_mapper.map(data, dob=individual.date_of_birth) + + # if ontology_routing is active + routed_diseases = get_routed_disease_dicts(data) + if routed_diseases: + logger.debug( + f"Processing {len(routed_diseases)} ontology-routed disease(s)" + ) + for routed_dict in routed_diseases: + try: + routed_proc, _ = create_processor("diseases") + routed_data = _inject_routed_disease( + data, routed_dict, routed_proc + ) + routed_disease_mapper = DiseaseMapper(routed_proc) + routed_result = routed_disease_mapper.map( + routed_data, dob=individual.date_of_birth + ) + if routed_result: + diseases.extend( + routed_result + if isinstance(routed_result, list) + else [routed_result] + ) + logger.debug( + f"Added routed disease from " + f"{routed_dict.get('__source_instrument')} " + f"#{routed_dict.get('__source_instance')}" + ) + except Exception as e: + logger.error( + f"Error processing routed disease " + f"({routed_dict.get('__source_instrument')} " + f"#{routed_dict.get('__source_instance')}): {e}" + ) + if debug: + logger.debug(traceback.format_exc()) + if debug: logger.debug(f"Total diseases: {len(diseases)}") - # --- Genetics: Variation Descriptor and Interpretations --- + # ── Genetics ───────────────────────────────────────────────────────── var_processor, _ = create_processor("variationDescriptor") variation_mapper = VariationDescriptorMapper(var_processor) variation_descriptors = variation_mapper.map(data) @@ -288,10 +377,10 @@ def create_processor(block: str, required: bool = False): interpretations = interpretation_mapper.map( data, subject_id=individual.id, - variation_descriptors=variation_descriptors + variation_descriptors=variation_descriptors, ) - # --- Metadata --- + # ── Metadata ────────────────────────────────────────────────────────── metadata_config = mapping_configs.get("metadata", {}) or {} code_systems = metadata_config.get("code_systems") if not code_systems: @@ -311,15 +400,17 @@ def create_processor(block: str, required: bool = False): variation_descriptors=variation_descriptors, ) if debug: - logger.debug(f"[metadata] used CURIE prefixes: {sorted(used_prefixes)}") - + logger.debug( + f"[metadata] used CURIE prefixes: {sorted(used_prefixes)}" + ) + metadata = MetadataMapper(None).map( - data={}, + data={}, created_by=created_by, code_systems=code_systems, - used_prefixes=used_prefixes, + used_prefixes=used_prefixes, ) - + phenopacket = Phenopacket( id=record_id, subject=individual, @@ -328,10 +419,12 @@ def create_processor(block: str, required: bool = False): diseases=diseases, medical_actions=medical_actions, meta_data=metadata, - interpretations=interpretations + interpretations=interpretations, ) if debug: - logger.debug(f"Successfully created phenopacket for record {record_id}") + logger.debug( + f"Successfully created phenopacket for record {record_id}" + ) return phenopacket except Exception as e: @@ -339,3 +432,92 @@ def create_processor(block: str, required: bool = False): if debug: logger.error(traceback.format_exc()) raise + + +# --------------------------------------------------------------------------- +# Private helpers — only used by create_phenopacket +# --------------------------------------------------------------------------- + +def _build_routed_pf_config( + base_config: Dict[str, Any], + routed_elements: list, +) -> Dict[str, Any]: + """ + Build a minimal phenotypicFeatures mapping config for HP-routed elements. + + The config inherits onset/severity/modifier field-name conventions from + the first existing phenotypicFeatures block so the PhenotypicFeatureMapper + knows which fields to read from the routed elements. + + The instrument name is set to the instrument of the first routed element, + so the mapper looks in the right sub-dict of each repeated_elements entry. + """ + # Detect the instrument from the first routed element + instrument = ( + routed_elements[0].get("redcap_repeat_instrument", "__routed__") + if routed_elements + else "__routed__" + ) + + # Keys to inherit from the base config's mapping_block + _INHERIT = { + "onset_date_field", "onset_date_fields", "onset_age_field", + "resolution_field", "severity_field", "evidence_field", + "modifier_temp_pattern_field", + *(f"modifier_field_{i}" for i in range(1, 10)), + } + + base_block = ( + base_config.get("mapping_block", {}) + if isinstance(base_config, dict) + else {} + ) + inherited = {k: v for k, v in base_block.items() if k in _INHERIT} + + return { + "instrument_name": instrument, + "mapping_block": { + "redcap_repeat_instrument": instrument, + "multi_onset": base_block.get("multi_onset", False), + **inherited, + }, + "enum_classes": ( + base_config.get("enum_classes", {}) + if isinstance(base_config, dict) + else {} + ), + "data_model": ( + base_config.get("data_model", "") + if isinstance(base_config, dict) + else "" + ), + } + + +def _inject_routed_disease( + data: Dict[str, Any], + routed_dict: Dict[str, Any], + processor, +) -> Dict[str, Any]: + """ + Produce a data view that makes the normalized routed_dict visible to + the DiseaseMapper via its normal field-access path. + + The DiseaseMapper reads ``term_field_1`` and ``onset_date_field`` from + the processor's mapping_config, then looks them up against the record + data. We temporarily override those keys in the processor config to + point at fixed sentinel field names, then return a synthetic data dict + that contains those sentinel values. + + This approach requires no changes to DiseaseMapper itself. + """ + # Overlay the routed values directly onto the processor's mapping config + # using the term_field_1 / onset_date_field convention. + processor.mapping_config["term_field_1"] = "__routed_term__" + processor.mapping_config["onset_date_field"] = "__routed_onset__" + + # Build a synthetic record that DiseaseMapper can traverse normally + synthetic = dict(data) + synthetic["__routed_term__"] = routed_dict.get("term_field_1") + synthetic["__routed_onset__"] = routed_dict.get("onset_date_field") + return synthetic \ No newline at end of file diff --git a/src/rarelink/phenopackets/validate.py b/src/rarelink/phenopackets/validate.py index 6e9fa62d9..7736a1056 100644 --- a/src/rarelink/phenopackets/validate.py +++ b/src/rarelink/phenopackets/validate.py @@ -1,3 +1,4 @@ +# src/rarelink/phenopackets/validate.py import json import re import shutil @@ -6,17 +7,14 @@ from typing import List, Tuple, Union import logging +from rarelink.phenopackets.adapter.ontology_routing_adapter import ( + check_prefix_placement, +) + logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Required top-level keys for a Phenopacket v2 JSON document -# --------------------------------------------------------------------------- _REQUIRED_TOP_LEVEL = {"id", "metaData"} - -# Required keys inside metaData _REQUIRED_METADATA = {"created", "createdBy", "phenopacketSchemaVersion"} - -# Basic pattern for CURIE-style ontology term IDs (e.g. HP:0001250, MONDO:0007843) _CURIE_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_]*:[A-Za-z0-9._\-]+$") @@ -24,23 +22,28 @@ def validate_phenopackets( path: Path, ) -> Union[Tuple[bool, str], List[Tuple[bool, str]]]: """ - Validates a phenopacket file or directory of phenopackets. + Validate a phenopacket file or directory of phenopackets. - Validation is performed in two stages: - 1. **Python-native checks** — always available, no external tooling needed. - Verifies JSON structure, required fields, and ontology term format. - 2. **phenopacket-tools CLI** — used automatically when the CLI is on PATH, - providing deeper schema-level validation on top of the Python checks. + Runs up to three stages: + + 1. **Python-native structural checks** — required fields, schema version, + CURIE format, ``subject.id``. + 2. **Ontology-prefix placement checks** — soft warnings when HP: terms + appear outside ``phenotypicFeatures`` or MONDO: terms appear outside + ``diseases``. Non-fatal: the phenopacket still passes, but warnings + are surfaced in the return string to help catch routing errors. + 3. **phenopacket-tools CLI** — when the CLI is on PATH, provides deeper + schema-level validation on top of stages 1 and 2. Args: - path (Path): Path to a single ``.json`` file or a directory of them. + path: Path to a single ``.json`` file or a directory. Returns: - - Single file → ``(bool, str)`` — (passed, detail message) - - Directory → ``List[(bool, str)]`` — one tuple per file + - Single file → ``(bool, str)`` + - Directory → ``List[(bool, str)]`` Raises: - ValueError: If the path does not exist, is not a JSON file, or a + ValueError: If path does not exist, is not a JSON file, or a directory contains no JSON files. """ logger.info("Starting validation of phenopackets...") @@ -55,7 +58,8 @@ def validate_phenopackets( if path.is_dir(): results = [ - _validate_single_phenopacket(fp) for fp in sorted(path.glob("*.json")) + _validate_single_phenopacket(fp) + for fp in sorted(path.glob("*.json")) ] if not results: raise ValueError(f"Directory {path} contains no JSON files.") @@ -68,51 +72,50 @@ def validate_phenopackets( raise ValueError(f"Path {path} is neither a file nor a directory.") -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - def _validate_single_phenopacket(file_path: Path) -> Tuple[bool, str]: """Run all available validation stages on one file.""" - logger.info(f"Validating {file_path}...") + logger.debug(f"Validating {file_path}...") - # --- Stage 1: Python-native structural validation --- + # Stage 1: structural ok, msg = _python_validate(file_path) if not ok: - logger.error(f"Structural validation failed for {file_path}: {msg}") + logger.warning(f"Structural validation failed for {file_path}: {msg}") return False, msg - # --- Stage 2: phenopacket-tools CLI (optional) --- + # Stage 2: prefix placement warnings (non-fatal) + prefix_warnings = _prefix_placement_check(file_path) + + # Stage 3: CLI (optional) if shutil.which("phenopacket-tools"): - ok, msg = _cli_validate(file_path) + ok, cli_msg = _cli_validate(file_path) if not ok: - logger.warning(f"CLI validation failed for {file_path}: {msg}") - return False, msg - logger.info(f"CLI validation passed for {file_path}") + logger.warning(f"CLI validation failed for {file_path}: {cli_msg}") + detail = cli_msg + if prefix_warnings: + detail += "\n\nOntology prefix warnings:\n" + "\n".join( + f" ⚠ {w}" for w in prefix_warnings + ) + return False, detail + logger.debug(f"CLI validation passed: {file_path}") else: logger.debug( - "phenopacket-tools CLI not found on PATH — skipping CLI validation. " - "Install it for deeper schema checks: " + "phenopacket-tools not found on PATH — skipping CLI validation. " + "Install for deeper schema checks: " "https://github.com/phenopackets/phenopacket-tools" ) - logger.info(f"Validation passed: {file_path}") - return True, f"OK: {file_path}" + detail = f"OK: {file_path}" + if prefix_warnings: + detail += "\n\nOntology prefix warnings (non-fatal):\n" + "\n".join( + f" ⚠ {w}" for w in prefix_warnings + ) + + logger.debug(f"Validation passed: {file_path}") + return True, detail def _python_validate(file_path: Path) -> Tuple[bool, str]: - """ - Lightweight Python-native checks against the Phenopacket v2 JSON schema: - - - Valid, parseable JSON - - Required top-level keys present (``id``, ``metaData``) - - ``metaData`` contains ``created``, ``createdBy``, ``phenopacketSchemaVersion`` - - ``phenopacketSchemaVersion`` is ``"2.0"`` - - All ontology terms (wherever ``{"id": ..., "label": ...}`` objects appear) - have a CURIE-formatted ``id`` - - ``subject.id`` present when ``subject`` block exists - """ - # 1. Parse JSON + """Python-native structural checks against the Phenopacket v2 JSON schema.""" try: with open(file_path) as f: doc = json.load(f) @@ -121,12 +124,10 @@ def _python_validate(file_path: Path) -> Tuple[bool, str]: errors: List[str] = [] - # 2. Required top-level keys missing = _REQUIRED_TOP_LEVEL - doc.keys() if missing: errors.append(f"Missing required top-level keys: {sorted(missing)}") - # 3. metaData checks meta = doc.get("metaData", {}) if isinstance(meta, dict): missing_meta = _REQUIRED_METADATA - meta.keys() @@ -137,69 +138,68 @@ def _python_validate(file_path: Path) -> Tuple[bool, str]: schema_ver = meta.get("phenopacketSchemaVersion", "") if schema_ver and not schema_ver.startswith("2"): errors.append( - f"phenopacketSchemaVersion is '{schema_ver}', expected '2.0'" + f"phenopacketSchemaVersion is '{schema_ver}', expected '2.x'" ) - # 4. subject.id subject = doc.get("subject") if subject is not None: if not isinstance(subject, dict) or not subject.get("id"): errors.append("subject block is present but missing 'id'") - # 5. Ontology term CURIE format — walk the whole document - curie_errors = _check_curie_terms(doc, path="$") - errors.extend(curie_errors) + errors.extend(_check_curie_terms(doc, path="$")) if errors: - detail = "; ".join(errors) - return False, detail - + return False, "; ".join(errors) return True, "Structural checks passed" +def _prefix_placement_check(file_path: Path) -> List[str]: + """Load the phenopacket JSON and run ontology-prefix placement checks.""" + try: + with open(file_path) as f: + doc = json.load(f) + return check_prefix_placement(doc) + except Exception as exc: + logger.debug( + f"Could not run prefix-placement check on {file_path}: {exc}" + ) + return [] + + +def _cli_validate(file_path: Path) -> Tuple[bool, str]: + """Run ``phenopacket-tools validate`` and return (success, output).""" + command = ["phenopacket-tools", "validate", str(file_path)] + try: + output = subprocess.check_output( + command, stderr=subprocess.STDOUT, text=True + ) + return True, output + except subprocess.CalledProcessError as exc: + return False, exc.output or str(exc) + + def _check_curie_terms(node, path: str) -> List[str]: """ - Recursively walk *node* and collect CURIE-format violations for any - object that has an ``"id"`` key whose sibling is ``"label"`` — i.e. an - OntologyClass-like object. + Recursively check that all ontology term ids are valid CURIEs. + An OntologyClass-like object is any dict with both ``"id"`` and ``"label"``. """ errors: List[str] = [] - if isinstance(node, dict): - # Looks like an OntologyClass if it has both "id" and "label" if "id" in node and "label" in node: term_id = node["id"] if isinstance(term_id, str) and not _CURIE_PATTERN.match(term_id): errors.append( f"Ontology term id '{term_id}' at {path} " - f"is not a valid CURIE (expected format PREFIX:localid)" + f"is not a valid CURIE (expected PREFIX:localid)" ) for key, value in node.items(): errors.extend(_check_curie_terms(value, path=f"{path}.{key}")) - elif isinstance(node, list): for i, item in enumerate(node): errors.extend(_check_curie_terms(item, path=f"{path}[{i}]")) - return errors -def _cli_validate(file_path: Path) -> Tuple[bool, str]: - """Run ``phenopacket-tools validate`` and return (success, output).""" - command = ["phenopacket-tools", "validate", str(file_path)] - try: - output = subprocess.check_output( - command, stderr=subprocess.STDOUT, text=True - ) - return True, output - except subprocess.CalledProcessError as exc: - return False, exc.output or str(exc) - - -# --------------------------------------------------------------------------- -# CLI entry-point -# --------------------------------------------------------------------------- - if __name__ == "__main__": import argparse import sys From 703cba2ae6f40fb3302442a0d7827db7399e5e1c Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:52:00 +0200 Subject: [PATCH 14/28] added chaching --- .../phenopackets/mappings/base_mapper.py | 23 ++----------------- submodules/tofhir | 1 - 2 files changed, 2 insertions(+), 22 deletions(-) delete mode 160000 submodules/tofhir diff --git a/src/rarelink/phenopackets/mappings/base_mapper.py b/src/rarelink/phenopackets/mappings/base_mapper.py index 3dabfa32c..cf2c87d83 100644 --- a/src/rarelink/phenopackets/mappings/base_mapper.py +++ b/src/rarelink/phenopackets/mappings/base_mapper.py @@ -4,8 +4,8 @@ from rarelink.utils.processor import DataProcessor from rarelink.utils.field_access import get_multi_instrument_field_value import rarelink.utils.label_fetching as labels +from functools import lru_cache -# Define type variable for the return type of mappers T = TypeVar('T') logger = logging.getLogger(__name__) @@ -44,20 +44,12 @@ def map(self, data: Dict[str, Any], **kwargs) -> Union[List[T], T, None]: """ try: - # Extract mapping configuration from processor config = self.processor.mapping_config - - # Determine if this is a single or multi-entity mapper is_multi = config.get("multi_entity", False) - - # Extract instruments for field access instruments = self._get_instruments(config) - - # Store the instruments in the processor for field access if instruments: config["all_instruments"] = instruments - # Map based on single or multi-entity configuration if is_multi: result = self._map_multi_entity(data, instruments, **kwargs) if result is None: @@ -70,7 +62,6 @@ def map(self, data: Dict[str, Any], **kwargs) -> Union[List[T], T, None]: logger.error(f"Error in {self.__class__.__name__}.map: {e}") import traceback logger.debug(traceback.format_exc()) - # Return empty list for multi-entity mappers, None for single-entity mappers return [] if config.get("multi_entity", False) else None def _map_single_entity( @@ -121,14 +112,12 @@ def get_field(self, Returns: Any: Field value or default """ - # Get the field path from the configuration field_path = self.processor.mapping_config.get(field_name) if not field_path: if self.debug_mode: logger.debug(f"Field '{field_name}' not found in mapping config") return default - # Use multi-instrument field access if instruments provided if instruments: value = get_multi_instrument_field_value( data=data, @@ -139,7 +128,6 @@ def get_field(self, if value is not None: return value - # Fallback to processor's get_field method return self.processor.get_field(data, field_name, default) def safe_execute(self, @@ -177,6 +165,7 @@ def process_code(self, code: str) -> str: """Process a code using the processor""" return self.processor.process_code(code) + @lru_cache(maxsize=None) def fetch_label(self, code: str, enum_class: Any = None) -> Optional[str]: """ Fetch a label with a single, patchable entrypoint: @@ -188,7 +177,6 @@ def fetch_label(self, code: str, enum_class: Any = None) -> Optional[str]: if not code: return None - # Resolve enum class if provided as a name enum_obj = None if isinstance(enum_class, str): enum_obj = getattr( @@ -196,20 +184,16 @@ def fetch_label(self, code: str, enum_class: Any = None) -> Optional[str]: else: enum_obj = enum_class - # Flatten mapping-config label dicts (support dict-of-dicts or flat dict) merged_dict: Dict[str, str] = {} mapping_config = self.processor.mapping_config or {} label_dicts = mapping_config.get("label_dicts") or {} if isinstance(label_dicts, dict): - # If it's a dict-of-dicts, merge values for v in label_dicts.values(): if isinstance(v, dict): merged_dict.update(v) - # If the dict is already flat (values not dicts), also merge as-is if all(not isinstance(v, dict) for v in label_dicts.values()): merged_dict.update(label_dicts) - # Delegate to the shared function (this is what the CLI patches) return labels.fetch_label(code, enum_class=enum_obj, label_dict=merged_dict) def fetch_mapping_value( @@ -231,19 +215,16 @@ def _get_instruments(self, config: Dict[str, Any] = None) -> List[str]: config = config or self.processor.mapping_config instruments = [] - # Get instrument_name(s) instrument_name = config.get("instrument_name") if isinstance(instrument_name, (list, set)): instruments.extend(list(instrument_name)) elif instrument_name: instruments.append(instrument_name) - # Add redcap_repeat_instrument if present repeat_instrument = config.get("redcap_repeat_instrument") if repeat_instrument and repeat_instrument not in instruments: instruments.append(repeat_instrument) - # Filter out dummy instruments return [i for i in instruments if i and i != "__dummy__"] def map_genetics_to_geno_ontology(self, diff --git a/submodules/tofhir b/submodules/tofhir deleted file mode 160000 index 54f4c3af3..000000000 --- a/submodules/tofhir +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 54f4c3af39d0904fcd69870c96e6fa60cc8f8d04 From 2194cc96717104b58297e703f5565555728ed751 Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 18:52:08 +0200 Subject: [PATCH 15/28] updated toml to 2.0.6 and clean up --- pyproject.toml | 4 ++-- src/rarelink/rd_cdm/codegen.py | 2 -- src/rarelink/rd_cdm/schema_template.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b57899efa..3df6e45b7 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rarelink" -version = "2.0.5" +version = "2.0.6" description = "RareLink - A REDCap-based Python framework linking international registries to HL7 FHIR and GA4GH Phenopackets" readme = "README.md" requires-python = ">=3.10, <3.13" @@ -53,7 +53,7 @@ dependencies = [ "schema-automator", "python-dotenv", "docker", - "rd-cdm>=2.0.2", + "rd-cdm>=2.0.3", ] [project.optional-dependencies] diff --git a/src/rarelink/rd_cdm/codegen.py b/src/rarelink/rd_cdm/codegen.py index 2971dc4bd..6928943ad 100644 --- a/src/rarelink/rd_cdm/codegen.py +++ b/src/rarelink/rd_cdm/codegen.py @@ -1,4 +1,3 @@ -# src/rarelink/rd_cdm/codegen.py from __future__ import annotations from pathlib import Path import importlib.resources as ilr @@ -145,7 +144,6 @@ def generate_python_classes(schema_path: Path, out_pkg_dir: Path) -> Path: f.write(code) return mod_path -# ---------- NEW: clone previous tree and update ---------- def _detect_prev_version(root: Path, target_version: str | None = None) -> str | None: """Pick the highest existing version under root that is NOT the target.""" candidates = sorted( diff --git a/src/rarelink/rd_cdm/schema_template.py b/src/rarelink/rd_cdm/schema_template.py index 3c10627e2..1367e7fcf 100644 --- a/src/rarelink/rd_cdm/schema_template.py +++ b/src/rarelink/rd_cdm/schema_template.py @@ -1,4 +1,3 @@ -# src/rarelink/cdm/schema_template.py from __future__ import annotations from typing import Dict, Any From 81e9f93bc71bf7cefb7b121b9a9516d8892019db Mon Sep 17 00:00:00 2001 From: Adam Graefe Date: Sun, 29 Mar 2026 19:27:13 +0200 Subject: [PATCH 16/28] updated codes according to rd-cdmv2.0.3 (specifically update of sex of birth to loinc_76689_9 in all files) and version updates --- .../2_rarelink_framework/2_2_rarelink_cdm.rst | 2 +- docs/4_user_guide/4_2_import_mapper.rst | 10 +- .../2_2_rarelink_cdm.html | 4 +- .../html/4_user_guide/4_2_import_mapper.html | 10 +- .../rarelink_cdm_datadictionary - v2_0_5.csv | 2 +- .../2_2_rarelink_cdm.rst.txt | 2 +- .../4_user_guide/4_2_import_mapper.rst.txt | 10 +- .../rarelink_cdm_datadictionary - v2_0_5.csv | 2 +- .../import_mapper_tsvs/example_1_sssom.tsv | 10 +- .../rarelink_cdm_sssom_template.tsv | 2 +- docs/_build/html/searchindex.js | 2 +- ... rarelink_cdm_datadictionary - v2_0_6.csv} | 4 +- .../import_mapper_tsvs/example_1_sssom.tsv | 10 +- .../rarelink_cdm_sssom_template.tsv | 2 +- ...evaluation_cohort_rarelink-cdm-linkml.json | 20 +- .../redcap/evaluation_cohort_redcap.json | 276 +++++++++--------- .../mappings/phenopackets/individual.py | 2 +- .../redcap/map_2_personal_information.py | 2 +- .../mappings/redcap/template.json | 2 +- .../rarelink_2_personal_information.py | 10 +- .../python_datamodel/rarelink_cdm.py | 10 +- ... rarelink_cdm_datadictionary - v2_0_6.csv} | 4 +- .../rarelink_2_personal_information.yaml | 10 +- .../rarelink_6_4_family_history.yaml | 2 +- src/rarelink/tofhir/mappings/ips.patient.json | 22 +- .../Rarelink_2_personal_information.json | 4 +- .../sample_records_rarelink_cdm.json | 6 +- .../sample_records_rarelink_cdm_1.json | 2 +- tests/utils/test_processor.py | 4 +- 29 files changed, 222 insertions(+), 226 deletions(-) rename docs/_static/{rarelink_cdm_datadictionary - v2_0_5.csv => rarelink_cdm_datadictionary - v2_0_6.csv} (99%) rename src/rarelink/rarelink_cdm/{rarelink_cdm_datadictionary - v2_0_5.csv => rarelink_cdm_datadictionary - v2_0_6.csv} (99%) diff --git a/docs/2_rarelink_framework/2_2_rarelink_cdm.rst b/docs/2_rarelink_framework/2_2_rarelink_cdm.rst index a6d3d3f07..bd4807d16 100755 --- a/docs/2_rarelink_framework/2_2_rarelink_cdm.rst +++ b/docs/2_rarelink_framework/2_2_rarelink_cdm.rst @@ -315,7 +315,7 @@ ______________________________________________________________ - Cardinality: Required (1..1) - Validation: Date format (YYYY-MM-DD) -- 2.2 Sex at Birth (``snomedct_281053000``) +- 2.2 Sex at Birth (``loinc_76689_9``) - Cardinality: Optional - Validation: Dropdown (choices encoded according to :ref:`1_5`) diff --git a/docs/4_user_guide/4_2_import_mapper.rst b/docs/4_user_guide/4_2_import_mapper.rst index 0dc70de95..6ce11f479 100755 --- a/docs/4_user_guide/4_2_import_mapper.rst +++ b/docs/4_user_guide/4_2_import_mapper.rst @@ -187,7 +187,7 @@ The template contains the following columns: RareLink-CDM schema, for example: - ``rarelink_cdm:snomedct_184099003`` for 2.1 Date of birth, or - - ``rarelink_cdm:snomedct_281053000`` for 2.2 Sex at birth. + - ``rarelink_cdm:loinc_76689_9`` for 2.2 Sex at birth. - **predicate_label**: The title of the target element from the schema, i.e. the *name* of the element @@ -198,7 +198,7 @@ The template contains the following columns: The target value in the RareLink-CDM. For literal fields, such as dates, this is the actual value (e.g., "1970-02-15"). For coded fields, such as sex, the object_id is the standardized code from the value set - (e.g., "SNOMEDCT:248152002" for female) or the code from the code system defined + (e.g., "LOINC:76689-9" for female) or the code from the code system defined by the data element (e.g., "HP:0001250" for a phenotype in element 6.2.1). - **object_label**: The human-readable label for the target value either from the @@ -255,7 +255,7 @@ Below you can find a **snippet of the RareLink-CDM SSSOM Template File:** # # === Personal Information (Singular) === "" rarelink_cdm:snomedct_184099003 "2.1 Date of birth" - lexical manual - "" rarelink_cdm:snomedct_281053000 "2.2 Sex at birth" lexical manual + "" rarelink_cdm:loinc_76689_9 "2.2 Sex at birth" lexical manual "" rarelink_cdm:snomedct_1296886006 "2.3 Karyotypic Sex" lexical manual "" rarelink_cdm:snomedct_263493000 "2.4 Gender Identity" lexical manual "" rarelink_cdm:snomedct_370159000 "2.5 Country of birth" lexical manual @@ -447,7 +447,7 @@ schema. subject_id subject_label predicate_id predicate_label object_id object_label match_type mapping_date mapping_tool ADR001 "Patient ADR001" rarelink_cdm:snomedct_422549004 "1.1 Pseudonym" ADR001 ADR001 lexical 2025-01-01 manual ADR001 "Patient ADR001" rarelink_cdm:snomedct_184099003 "2.1 Date of birth" 1970-02-15 - lexical 2025-01-01 manual - ADR001 "Patient ADR001" rarelink_cdm:snomedct_281053000 "2.2 Sex at birth" SNOMEDCT:248152002 Female lexical 2025-01-01 manual + ADR001 "Patient ADR001" rarelink_cdm:loinc_76689_9 "2.2 Sex at birth" LOINC:76689-9 Female lexical 2025-01-01 manual ADR001 "Patient ADR001" rarelink_cdm:snomedct_263493000 "2.4 Gender Identity" SNOMEDCT:446141000124107 Female gender identity lexical 2025-01-01 manual ADR001 "Patient ADR001" rarelink_cdm:snomedct_278844005 "3.1 Vital Status" SNOMEDCT:438949009 Alive lexical 2025-01-01 manual ADR001 "Patient ADR001" rarelink_cdm:snomedct_64572001_mondo "5.1 Disease [MONDO]" MONDO:0012345 Wilson Disease lexical 2025-01-01 manual @@ -475,7 +475,7 @@ schema. # Mappings for a second patient (ADR002) ADR002 "Patient ADR002" rarelink_cdm:snomedct_422549004 "1.1 Pseudonym" ADR002 ADR002 lexical 2025-01-01 manual ADR002 "Patient ADR002" rarelink_cdm:snomedct_184099003 "2.1 Date of birth" 1982-07-30 1982-07-30 lexical 2025-01-01 manual - ADR002 "Patient ADR002" rarelink_cdm:snomedct_281053000 "2.2 Sex at birth" SNOMEDCT:248153007 Male lexical 2025-01-01 manual + ADR002 "Patient ADR002" rarelink_cdm:loinc_76689_9 "2.2 Sex at birth" SNOMEDCT:248153007 Male lexical 2025-01-01 manual ADR002 "Patient ADR002" rarelink_cdm:snomedct_278844005 "3.1 Vital Status" SNOMEDCT:438949009 Alive lexical 2025-01-01 manual ADR002 "Patient ADR002" rarelink_cdm:snomedct_64572001_mondo "5.1 Disease [MONDO]" MONDO:0010526 Fabry Disease lexical 2025-01-01 manual ADR002 "Patient ADR002" rarelink_cdm:loinc_99498_8 "5.2 Verification status" HL7FHIR:differential Differential lexical 2025-01-01 manual diff --git a/docs/_build/html/2_rarelink_framework/2_2_rarelink_cdm.html b/docs/_build/html/2_rarelink_framework/2_2_rarelink_cdm.html index c6a011ac0..5ba77d0b8 100644 --- a/docs/_build/html/2_rarelink_framework/2_2_rarelink_cdm.html +++ b/docs/_build/html/2_rarelink_framework/2_2_rarelink_cdm.html @@ -268,7 +268,7 @@

Contents

REDCap Data Dictionary

The RareLink CDM Data Dictionary is available for download as a CSV file:

-

Download: RareLink-CDM Data Dictionary (v2.0.5)

+

Download: RareLink-CDM Data Dictionary (v2.0.5)

… Next steps