From 31f93c102db743ad2fbe1a1387ee9858f94dc648 Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sun, 17 May 2026 09:29:59 +0100 Subject: [PATCH 1/7] setup prod environment locally --- medcat-trainer/docker-compose-prod.yml | 2 +- medcat-trainer/envs/env-prod | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/medcat-trainer/docker-compose-prod.yml b/medcat-trainer/docker-compose-prod.yml index b2c90a3d1..4159de940 100644 --- a/medcat-trainer/docker-compose-prod.yml +++ b/medcat-trainer/docker-compose-prod.yml @@ -5,7 +5,7 @@ services: container_name: medcattrainer hostname: medcat restart: always - image: cogstacksystems/medcat-trainer:v3.4.1 + image: cogstacksystems/medcat-trainer:latest volumes: - api-media:/home/api/media - api-static:/home/api/static diff --git a/medcat-trainer/envs/env-prod b/medcat-trainer/envs/env-prod index 39bd1ff9f..a69245204 100644 --- a/medcat-trainer/envs/env-prod +++ b/medcat-trainer/envs/env-prod @@ -8,10 +8,10 @@ MAX_MEDCAT_MODELS=2 ENV=prod # SECRET KEY - edit this for prod deployments, -# SECRET_KEY= +SECRET_KEY=QNh/dY8k1U9x/XZpAIv28p1qN2PsOq1tgIwcGqrlTKExrGOx3RulQILep6LGAVvJpJM= # Complete once this is deployed -CSRF_TRUSTED_ORIGINS= +CSRF_TRUSTED_ORIGINS=http://localhost:8001,http://127.0.0.1:8001 # Django Debug mode should be False for prod DEBUG=0 From 79bd7bec7d0f9f932754f918adc6ea44179618e4 Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sat, 20 Jun 2026 22:01:24 +0100 Subject: [PATCH 2/7] changed back to version 3.6.0 due meta-annotation webpage not working --- medcat-trainer/docker-compose-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medcat-trainer/docker-compose-prod.yml b/medcat-trainer/docker-compose-prod.yml index 4159de940..e4a45fa8b 100644 --- a/medcat-trainer/docker-compose-prod.yml +++ b/medcat-trainer/docker-compose-prod.yml @@ -5,7 +5,7 @@ services: container_name: medcattrainer hostname: medcat restart: always - image: cogstacksystems/medcat-trainer:latest + image: cogstacksystems/medcat-trainer:3.6.0 volumes: - api-media:/home/api/media - api-static:/home/api/static @@ -16,7 +16,7 @@ services: env_file: - ./envs/env-prod environment: - - MCT_VERSION=v3.4.1 + - MCT_VERSION=v3.6.0 command: /usr/bin/supervisord -c /etc/supervisord.conf nginx: From 0d3ec347e80b1f1debac0abd1f035a951e8999e0 Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sat, 27 Jun 2026 19:06:12 +0100 Subject: [PATCH 3/7] updated to medcattrainer 3.7.0 --- medcat-trainer/docker-compose-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medcat-trainer/docker-compose-prod.yml b/medcat-trainer/docker-compose-prod.yml index e4a45fa8b..d689c8122 100644 --- a/medcat-trainer/docker-compose-prod.yml +++ b/medcat-trainer/docker-compose-prod.yml @@ -5,7 +5,7 @@ services: container_name: medcattrainer hostname: medcat restart: always - image: cogstacksystems/medcat-trainer:3.6.0 + image: cogstacksystems/medcat-trainer:3.7.0 volumes: - api-media:/home/api/media - api-static:/home/api/static @@ -16,7 +16,7 @@ services: env_file: - ./envs/env-prod environment: - - MCT_VERSION=v3.6.0 + - MCT_VERSION=v3.7.0 command: /usr/bin/supervisord -c /etc/supervisord.conf nginx: From 0d6c432eeca79fe695a8f2672c016f48fb4de42a Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sat, 4 Jul 2026 23:40:16 +0100 Subject: [PATCH 4/7] fix(medcat): prevent silent document loss in TestTrainSplitter.split() --- medcat-v2/medcat/utils/data_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/medcat-v2/medcat/utils/data_utils.py b/medcat-v2/medcat/utils/data_utils.py index d4d5c355d..e39ab7a76 100644 --- a/medcat-v2/medcat/utils/data_utils.py +++ b/medcat-v2/medcat/utils/data_utils.py @@ -74,6 +74,7 @@ def split(self) -> tuple[MedCATTrainerExport, MedCATTrainerExport, for i_document in np.random.permutation(range(0, num_of_docs)): # Do we have enough documents in the test set if self.test_anns / self.total_anns >= self.test_size: + train_project['documents'].append(project['documents'][i_document]) continue document = project['documents'][i_document] self._split_doc_train_test(document, cui_filter, From 1f85ce78100a5b4c55634bc21724a67a922787fc Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Fri, 10 Jul 2026 23:58:31 +0100 Subject: [PATCH 5/7] fix(medcat): added test for TestTrainSplitter.split() document loss patch --- medcat-v2/tests/utils/test_data_utils.py | 155 +++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 medcat-v2/tests/utils/test_data_utils.py diff --git a/medcat-v2/tests/utils/test_data_utils.py b/medcat-v2/tests/utils/test_data_utils.py new file mode 100644 index 000000000..fcc6791a7 --- /dev/null +++ b/medcat-v2/tests/utils/test_data_utils.py @@ -0,0 +1,155 @@ +import unittest +import numpy as np + +from medcat.utils.data_utils import TestTrainSplitter, make_mc_train_test +from medcat.data.mctexport import MedCATTrainerExport + + +class _FakeCDB: + """Minimal CDB stub - TestTrainSplitter does not access CDB directly.""" + pass + + +def _make_synthetic_export(n_docs: int, + n_projects: int = 1, + cui: str = "428763004", + anns_per_doc: int = 1) -> MedCATTrainerExport: + """Create a synthetic MedCATTrainerExport with a single common CUI. + + Using a single CUI appearing in all documents replicates the real-world + condition that triggers the bug: the test quota fills quickly because + MAX_TEST_FRACTION (0.3) is reached after only a small number of documents, + causing all remaining documents to be processed by the if-branch in split(). + """ + docs_per_project = n_docs // n_projects + projects = [] + doc_id = 0 + + for p_idx in range(n_projects): + documents = [] + for d_idx in range(docs_per_project): + documents.append({ + "id": doc_id, + "name": f"note_{doc_id}", + "last_modified": "N/A", + "text": f"Patient has Staphylococcus aureus bacteraemia. Document {doc_id}.", + "annotations": [ + { + "cui": cui, + "value": "Staphylococcus aureus bacteraemia", + "start": 11, + "end": 44, + } + for _ in range(anns_per_doc) + ] + }) + doc_id += 1 + + projects.append({ + "id": p_idx, + "name": f"project_{p_idx}", + "cuis": "", + "documents": documents, + }) + + return {"projects": projects} + + +class TestTrainSplitterNoDocumentLossTests(unittest.TestCase): + """Tests that TestTrainSplitter assigns all documents to train or test. + + Previously, when the test annotation quota was reached, documents were + silently dropped via `continue` rather than being routed to the train set. + This caused the majority of documents to be discarded from both sets. + + See: https://discourse.cogstack.org/t/testtrainsplitter-silently-drops-documents + """ + + N_DOCS = 50 + TEST_SIZE = 0.2 + RNG_SEED = 42 + + @classmethod + def setUpClass(cls): + cls.cdb = _FakeCDB() + cls.export = _make_synthetic_export(cls.N_DOCS) + + def setUp(self): + np.random.seed(self.RNG_SEED) + + def _count_docs(self, dataset: MedCATTrainerExport) -> int: + return sum(len(p['documents']) for p in dataset['projects']) + + def test_no_documents_lost_single_project(self): + """All documents should be assigned to train or test — none dropped.""" + splitter = TestTrainSplitter(self.export, self.cdb, + test_size=self.TEST_SIZE) + train_set, test_set, _, _ = splitter.split() + + total = self._count_docs(self.export) + train = self._count_docs(train_set) + test = self._count_docs(test_set) + + self.assertEqual( + train + test, total, + f"Documents lost: {total - train - test} " + f"(train={train}, test={test}, total={total})" + ) + + def test_train_set_is_larger_than_test_set(self): + """Train set should contain the majority of documents.""" + splitter = TestTrainSplitter(self.export, self.cdb, + test_size=self.TEST_SIZE) + train_set, test_set, _, _ = splitter.split() + + train = self._count_docs(train_set) + test = self._count_docs(test_set) + + self.assertGreater(train, test, + f"Expected train ({train}) > test ({test})") + + def test_test_set_is_not_empty(self): + """Test set should contain at least one document.""" + splitter = TestTrainSplitter(self.export, self.cdb, + test_size=self.TEST_SIZE) + train_set, test_set, _, _ = splitter.split() + + test = self._count_docs(test_set) + self.assertGreater(test, 0, "Test set should not be empty") + + def test_no_documents_lost_multi_project(self): + """No documents lost when data spans multiple projects.""" + export = _make_synthetic_export(self.N_DOCS, n_projects=2) + splitter = TestTrainSplitter(export, self.cdb, + test_size=self.TEST_SIZE) + train_set, test_set, _, _ = splitter.split() + + total = self._count_docs(export) + train = self._count_docs(train_set) + test = self._count_docs(test_set) + + self.assertEqual( + train + test, total, + f"Documents lost across projects: {total - train - test} " + f"(train={train}, test={test}, total={total})" + ) + + def test_make_mc_train_test_no_documents_lost(self): + """make_mc_train_test wrapper should also lose no documents.""" + np.random.seed(self.RNG_SEED) + train_set, test_set, _, _ = make_mc_train_test( + self.export, self.cdb, test_size=self.TEST_SIZE + ) + + total = self._count_docs(self.export) + train = self._count_docs(train_set) + test = self._count_docs(test_set) + + self.assertEqual( + train + test, total, + f"make_mc_train_test lost {total - train - test} documents" + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From e6b7e23a52fa58accc990c6fd0665e1f93910876 Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sat, 11 Jul 2026 10:51:07 +0100 Subject: [PATCH 6/7] revert docker compose files to default --- medcat-trainer/docker-compose-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medcat-trainer/docker-compose-prod.yml b/medcat-trainer/docker-compose-prod.yml index d689c8122..b2c90a3d1 100644 --- a/medcat-trainer/docker-compose-prod.yml +++ b/medcat-trainer/docker-compose-prod.yml @@ -5,7 +5,7 @@ services: container_name: medcattrainer hostname: medcat restart: always - image: cogstacksystems/medcat-trainer:3.7.0 + image: cogstacksystems/medcat-trainer:v3.4.1 volumes: - api-media:/home/api/media - api-static:/home/api/static @@ -16,7 +16,7 @@ services: env_file: - ./envs/env-prod environment: - - MCT_VERSION=v3.7.0 + - MCT_VERSION=v3.4.1 command: /usr/bin/supervisord -c /etc/supervisord.conf nginx: From 418717b01a1508e3b7d9c70e1a05ceba00b29882 Mon Sep 17 00:00:00 2001 From: Ollie Lloyd Date: Sat, 11 Jul 2026 10:52:53 +0100 Subject: [PATCH 7/7] revert env-prod file to default --- medcat-trainer/envs/env-prod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/medcat-trainer/envs/env-prod b/medcat-trainer/envs/env-prod index a69245204..39bd1ff9f 100644 --- a/medcat-trainer/envs/env-prod +++ b/medcat-trainer/envs/env-prod @@ -8,10 +8,10 @@ MAX_MEDCAT_MODELS=2 ENV=prod # SECRET KEY - edit this for prod deployments, -SECRET_KEY=QNh/dY8k1U9x/XZpAIv28p1qN2PsOq1tgIwcGqrlTKExrGOx3RulQILep6LGAVvJpJM= +# SECRET_KEY= # Complete once this is deployed -CSRF_TRUSTED_ORIGINS=http://localhost:8001,http://127.0.0.1:8001 +CSRF_TRUSTED_ORIGINS= # Django Debug mode should be False for prod DEBUG=0