Improve NameRes documentation: Translator Guide, Babel data guide, and doc drift fixes

.github/workflows/release-nameres-loading.yml

@@ -17,9 +17,6 @@ jobs:
         steps:
             - name: Check out the repo
               uses: actions/checkout@v2
-            - name: Get the version
-              id: get_version
-              run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
             - name: Extract metadata (tags, labels) for Docker
               id: meta
               uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
@@ -44,3 +41,16 @@ jobs:
                   tags: ${{ steps.meta.outputs.tags }}
                   labels: ${{ steps.meta.outputs.labels }}
                   build-args: BRANCH_NAME=${{ github.event.release.target_commitish }}
+                  # Without a cache every run rebuilds the whole image: apt upgrade,
+                  # seven apt installs and a ~250MB Solr tarball fetched through an
+                  # Apache mirror, which is why a build takes ~25 minutes. Layers
+                  # before the first change now come from the GitHub Actions cache.
+                  # mode=max keeps intermediate layers too, so a change late in the
+                  # Dockerfile (the COPYs) does not re-download Solr.
+                  #
+                  # The trade is that `apt upgrade` is cached as well, so an image
+                  # built from an unchanged Dockerfile stops picking up new Debian
+                  # packages. That is the right way round here: this is short-lived
+                  # build tooling, and any Dockerfile edit invalidates the cache.
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max

.github/workflows/tester.yml

@@ -22,8 +22,19 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Run SOLR
-        run: docker run --name name_lookup -d -p 8983:8983 -t solr -DzkRun
+      - name: Run Solr (standalone)
+        run: docker run --name name_lookup -d -p 8983:8983 solr:9.10
+
+      - name: Create the name_lookup core from the checked-in configset
+        run: |
+          for _ in $(seq 60); do
+            curl -sf 'http://localhost:8983/solr/admin/cores?action=STATUS' > /dev/null && break
+            sleep 2
+          done
+          curl -sf 'http://localhost:8983/solr/admin/cores?action=STATUS' > /dev/null \
+            || { echo "Solr did not come up:"; docker logs name_lookup; exit 1; }
+          docker cp data-loading/configsets/name_lookup name_lookup:/tmp/name_lookup
+          docker exec name_lookup solr create -c name_lookup -d /tmp/name_lookup
 
       - name: Install dependencies
         run: |
@@ -38,3 +49,62 @@ jobs:
       - name: Run the tests
         run: |
           python -m pytest tests/
+
+      # The loader's guard is what stands between us and a silently half-loaded
+      # 130 GB index, so check that a bad file actually stops it. LOAD_APPEND=1
+      # because the core is loaded by now, and without it the loader would refuse
+      # for that reason instead -- the test would pass without testing anything.
+      - name: The loader must abort on a file that does not load
+        run: |
+          echo 'this is not json' > /tmp/malformed.json
+          if LOAD_APPEND=1 ./data-loading/setup-and-load-solr.sh /tmp/malformed.json; then
+            echo "Expected the loader to fail on a malformed file, but it succeeded."
+            exit 1
+          fi
+          echo "Loader aborted as expected."
+
+      # The other half of that guard: loading is additive and assigns fresh UUIDs, so
+      # a re-run over the same files doubles the index and the document-count check
+      # cannot see it (the delta is still exactly right). Refusing a non-empty core is
+      # the only thing that catches it.
+      - name: The loader must refuse a core that already has documents
+        run: |
+          if ./data-loading/setup-and-load-solr.sh tests/data/test-synonyms.json; then
+            echo "Expected the loader to refuse a non-empty core, but it loaded again."
+            exit 1
+          fi
+          echo "Loader refused the non-empty core as expected."
+
+      # The backup is the actual product of data-loading/, so check that a core can be
+      # tarred up, dropped into a fresh Solr, and served without any restore-time setup.
+      #
+      # The tar flags are the ones the Makefile uses, and the restore deliberately does
+      # NOT chown afterwards: the loading image runs as uid 1000 and the serving image
+      # as 8983, so if the tarball did not stamp the ownership the restored core would
+      # be unwritable and Solr would fail to take its write lock. chown-ing here would
+      # hide exactly the bug this is meant to catch. The chown to 1000 below stands in
+      # for the loading image, so the source ownership is wrong before we start.
+      - name: Verify the backup roundtrip (tar the core, restore it into a fresh Solr)
+        run: |
+          docker exec -u root name_lookup chown -R 1000:1000 /var/solr/data/name_lookup
+          docker exec -u root name_lookup tar -C /var/solr/data \
+            --numeric-owner --owner=8983 --group=8983 \
+            --exclude='name_lookup/data/tlog' \
+            -cf - name_lookup > snapshot.backup.tar
+          docker run --name name_lookup_restored -d -p 8984:8983 solr:9.10
+          for _ in $(seq 60); do
+            curl -sf 'http://localhost:8984/solr/admin/cores?action=STATUS' > /dev/null && break
+            sleep 2
+          done
+          docker cp snapshot.backup.tar name_lookup_restored:/tmp/snapshot.backup.tar
+          docker exec -u root name_lookup_restored tar -C /var/solr/data --numeric-owner -xf /tmp/snapshot.backup.tar
+          docker restart name_lookup_restored
+          for _ in $(seq 60); do
+            curl -sf 'http://localhost:8984/solr/name_lookup/admin/ping' > /dev/null && break
+            sleep 2
+          done
+          count=$(curl -sf 'http://localhost:8984/solr/name_lookup/query?q=*:*&rows=0' \
+            | grep -oE '"numFound":[[:space:]]*[0-9]+' | head -1 | grep -oE '[0-9]+')
+          expected=$(grep -c '[^[:space:]]' tests/data/test-synonyms.json)
+          echo "Restored core reports ${count:-0} documents; expected ${expected}."
+          [ "${count:-0}" = "${expected}" ]

CLAUDE.md

@@ -10,7 +10,12 @@ NameRes (Name Resolver) is a biomedical entity name resolution service that maps
 
 ### Running Tests
 ```bash
-# Load test data into Solr first (requires Solr running)
+# Start standalone Solr and create the name_lookup core from the checked-in configset
+docker run --name name_lookup -d -p 8983:8983 solr:9.10
+docker cp data-loading/configsets/name_lookup name_lookup:/tmp/name_lookup
+docker exec name_lookup solr create -c name_lookup -d /tmp/name_lookup
+
+# Load test data into the core (parallel load, with a document-count guard)
 ./data-loading/setup-and-load-solr.sh tests/data/test-synonyms.json
 
 # Run all tests
@@ -54,38 +59,56 @@ pip install -r requirements.txt
 4. Results are scored, normalized, and returned as JSON
 
 ### Key Files
-- `api/server.py` - Core FastAPI application (~717 lines): all endpoints, Pydantic models, Solr query construction, environment config
+- `api/server.py` - Core FastAPI application: all endpoints, Pydantic models, Solr query construction, environment config
 - `api/apidocs.py` - Custom OpenAPI schema construction
-- `api/resources/.openapi.yml` - OpenAPI 3.0.2 spec with service metadata
+- `api/resources/openapi.yml` - OpenAPI 3.0.2 spec with service metadata
 - `main.py` / `main.sh` - WSGI/ASGI entry points (port 2433)
 - `tests/test_service.py` - Integration tests using FastAPI `TestClient`
 - `tests/data/test-synonyms.json` - Test dataset for Solr
 
 ### Environment Variables
-- `SOLR_HOST` / `SOLR_PORT` - Solr connection (default: `localhost:8983`)
+See `documentation/Deployment.md` for the full list. The main ones:
+- `SOLR_HOST` / `SOLR_PORT` / `SOLR_CORE` - Solr connection (default: `localhost:8983`, core `name_lookup`)
 - `LOGLEVEL` - Logging level
-- `SERVER_ROOT` - API root path prefix
+- `SERVER_NAME` / `SERVER_ROOT` - Infores ID and API root path prefix
 - `MATURITY_VALUE` / `LOCATION_VALUE` - TRAPI metadata fields
+- `BABEL_VERSION` / `BABEL_VERSION_URL` / `BIOLINK_MODEL_TAG` - reported by `/status`; describe the
+  data the index was built from
+- `OTEL_ENABLED` / `JAEGER_*` - OpenTelemetry
 
 ### API Endpoints
 - `GET/POST /lookup` - Primary name-to-CURIE lookup with scoring
 - `POST /bulk-lookup` - Batch queries via `NameResQuery` model
-- `GET /reverse-lookup` - CURIE-to-names lookup
-- `POST /synonyms` - Get synonyms for a list of CURIEs
-- `POST /lookup-curies` - Filter existing CURIEs with type subsetting
-- `GET /status` - Health check with Solr document counts
+- `GET/POST /synonyms` - Get synonyms for a list of preferred CURIEs
+- `GET/POST /reverse_lookup` - Deprecated alias for `/synonyms`
+- `GET /status` - Health check with Solr document counts, plus the Babel and Biolink versions
 
 ### Data Model
 Solr documents contain: `curie`, `preferred_name`, `names` (synonym list), and biolink type information. Lookup results are `LookupResult` objects with scoring fields. Results are conflated using GeneProtein and DrugChemical conflation rules.
 
 ### Infrastructure
 - **Stateless API container** - Python 3.11.5/FastAPI
-- **Persistent Solr 9.1** - Data in volume-mounted `./data/solr`
+- **Persistent Solr 9.10 (standalone)** - Data in volume-mounted `./data/solr`
 - **Data loading** - Separate pipeline in `data-loading/` (Makefile-driven, also has Kubernetes configs)
 - **CI/CD** - GitHub Actions: runs tests on push, publishes Docker image to GitHub Packages on release
 
 ## Documentation
 - `documentation/API.md` - Endpoint reference
+- `documentation/Babel.md` - Where the data comes from, and the Babel behaviour visible through this
+  API. This is the only file that should link to a *specific file* inside the Babel repository; keep
+  new cross-repo links here so a reorganization there is a one-file fix.
 - `documentation/Deployment.md` - Docker/Kubernetes deployment guide
 - `documentation/Scoring.md` - Scoring algorithm details
-- `documentation/NameResolution.ipynb` - Interactive usage examples
+- `documentation/NameResolution.ipynb` - Interactive usage examples
+- `documentation/TranslatorGuide.md` - Translator-specific usage guidance
+
+### Linking to GitHub
+
+NameResolution, Babel and NodeNormalization all use `main` as their default branch. Babel's was
+renamed from `master` fairly recently and NameResolution has no `master` branch at all, so
+`/blob/master/` URLs resolve only through GitHub's post-rename redirect — they look fine until
+that redirect goes away. Always write `/blob/main/`.
+
+Do not check this against a local clone's `origin/HEAD`: that ref is cached at clone time and does
+not follow a remote rename, so it will still say `master` long after the rename. Use
+`gh repo view <owner>/<repo> --json defaultBranchRef`.

README.md

@@ -6,9 +6,30 @@ Name Resolver (Name Lookup or NameRes) takes lexical strings and attempts to map
 along with many other options. Given a preferred CURIE, the known synonyms of that CURIE can also be retrieved.
 Multiple results may be returned representing possible conceptual matches, but all of the identifiers have been correctly normalized using the [Node Normalization](https://github.com/NCATSTranslator/NodeNormalization) service.
 
-Note that the results returned by this service have been conflated using both GeneProtein and DrugChemical conflation; you can read more about this at the [Conflation documentation](https://github.com/NCATSTranslator/Babel/blob/master/docs/Conflation.md).
+The concepts NameRes searches are built by the [Babel](https://github.com/NCATSTranslator/Babel)
+pipeline. Note that the results returned by this service have been conflated using both GeneProtein
+and DrugChemical conflation; you can read more about this, and about the other Babel behaviour
+visible through this API, at [Where NameRes data comes from](documentation/Babel.md).
 
-* See this [Jupyter Notebook](documentation/NameResolution.ipynb) for examples of use.
-* See the [API documentation](documentation/API.md) for information about the NameRes API.
-* See [Scoring](documentation/Scoring.md) for information about the scoring algorithm used by NameRes.
-* See [Deployment](documentation/Deployment.md) for instructions on deploying NameRes.
+## Getting started
+
+The best place to start is the Jupyter Notebook, which walks through the most common use cases with live examples:
+
+* [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NCATSTranslator/NameResolution/blob/main/documentation/NameResolution.ipynb) [Jupyter Notebook](documentation/NameResolution.ipynb) — interactive examples covering lookup, filtering, autocomplete, bulk lookup, and synonyms
+
+## Documentation
+
+* [Translator Guide](documentation/TranslatorGuide.md) — what to do when results are unexpected, when to use `/synonyms` vs. NodeNorm, and performance tips
+* [API documentation](documentation/API.md) — full reference for all NameRes endpoints
+* [Where NameRes data comes from](documentation/Babel.md) — how Babel builds the concepts NameRes searches, and the gotchas that follow from that
+* [Scoring](documentation/Scoring.md) — how NameRes scores and ranks results
+* [Deployment](documentation/Deployment.md) — instructions for deploying NameRes
+
+## Reporting a problem
+
+If a *concept* is wrong -- identifiers merged that shouldn't be, a wrong Biolink type or preferred
+label, missing synonyms -- report it to [Babel](https://github.com/NCATSTranslator/Babel/issues),
+which builds the concepts. If the *service* is wrong -- errors, badly ranked results, a parameter
+not behaving as documented -- report it
+[here](https://github.com/NCATSTranslator/NameResolution/issues). If you're not sure, file it in
+Babel. See [Where NameRes data comes from](documentation/Babel.md#reporting-a-problem) for details.

api/apidocs.py

@@ -35,8 +35,10 @@ def construct_open_api_schema(app) -> Dict[str, str]:
     with open(Path(__file__).parent / 'resources' / 'openapi.yml', 'r') as apd_file:
         api_docs = load(apd_file, Loader=SafeLoader)
 
+    # Already built (FastAPI caches it on the app), so hand back the cached schema.
+    # This is a dict, not a callable -- calling it raises TypeError.
     if app.openapi_schema:
-        return app.openapi_schema()
+        return app.openapi_schema
 
     open_api_schema = get_openapi(
         title=api_docs['info']['title'],

api/resources/openapi.yml

@@ -1,7 +1,7 @@
 openapi: 3.0.2
 info:
   title: Name Resolver
-  version: 1.6.2
+  version: 1.7.0
   email: bizon@renci.org
   name: Chris Bizon
   x-id: https://github.com/cbizon
@@ -13,8 +13,11 @@ info:
     have been correctly normalized using the
     <a href="https://github.com/NCATSTranslator/NodeNormalization">Node Normalization</a> service.<p/>You can read more
     about this API on the <a href="https://github.com/NCATSTranslator/NameResolution">NameResolution GitHub repository</a>.<p/>
-    Note that the returned by this service have been conflated using both GeneProtein and DrugChemical conflation;
-    you can read more about this at the <a href="https://github.com/NCATSTranslator/Babel/blob/master/docs/Conflation.md">Conflation documentation</a>.'
+    <p>Note that the results returned by this service have been conflated using both GeneProtein and DrugChemical
+    conflation; the active conflations for any deployment can be discovered via the <code>/status</code> endpoint.
+    You can read more about this, and about the other <a href="https://github.com/NCATSTranslator/Babel">Babel</a>
+    behaviour visible through this API, at
+    <a href="https://github.com/NCATSTranslator/NameResolution/blob/main/documentation/Babel.md">Where NameRes data comes from</a>.</p>'
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT