Feature/graphql sources 20260906

core/__init__.py

@@ -4,7 +4,7 @@
 
 __all__ = ('celery_app',)
 
-API_VERSION = '2.3.201'
+API_VERSION = '2.3.202'
 API_BUILD = 'dev'
 VERSION = API_VERSION + '-' + API_BUILD
 __version__ = VERSION

core/common/permissions.py

@@ -42,16 +42,30 @@ def has_object_permission(self, request, view, obj):
         return False
 
 
-class CanViewConceptDictionary(HasPrivateAccess):
+def user_can_view_concept_dictionary(user, obj) -> bool:
+    """Share repository visibility without mixing user, organization and repository primary keys."""
+    if obj.public_access in [ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW]:
+        return True
+    if user.is_staff:
+        return True
+    if user.is_authenticated:
+        if getattr(obj, 'user_id', None) == user.id:
+            return True
+        organization_id = getattr(obj, 'organization_id', None)
+        if getattr(obj, 'resource_type', None) == 'Organization':
+            organization_id = obj.id
+        if organization_id and user.organizations.filter(id=organization_id).exists():
+            return True
+    return False
+
+
+class CanViewConceptDictionary(BasePermission):
     """
     The user can view this source
     """
 
     def has_object_permission(self, request, view, obj):
-        if obj.public_access in [ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW]:
-            return True
-
-        return super().has_object_permission(request, view, obj)
+        return user_can_view_concept_dictionary(request.user, obj)
 
 
 class CanEditConceptDictionary(HasPrivateAccess):

core/common/search.py

@@ -15,6 +15,83 @@
 
 from core.common.constants import ES_REQUEST_TIMEOUT
 from core.common.utils import is_url_encoded_string
+from core.orgs.constants import ORG_OBJECT_TYPE
+from core.users.constants import USER_OBJECT_TYPE
+
+
+def get_document_public_visibility_criteria(  # pylint: disable=too-many-arguments
+    user,
+    include_creator_private_access=False,
+    include_owner_private_access=False,
+    include_organization_memberships=False,
+):
+    """Return a shared Elasticsearch visibility criterion for owner-scoped documents.
+
+    The base criterion is always ``public_can_view=True``. Anonymous users get only that.
+    Authenticated users may additionally see private documents matched by the OR of the
+    enabled flags below — each flag widens visibility in a specific way:
+
+    - ``include_creator_private_access``: include private docs where ``created_by`` equals
+      the current user's username. Mirrors the historical REST concept/source-child rule
+      (a creator always sees their own private content). Used by REST list endpoints.
+
+    - ``include_owner_private_access``: include private docs owned by the user itself
+      (``owner_type=USER`` and ``owner=username``). Used by GraphQL to mirror how list APIs
+      expose a user's own private repositories.
+
+    - ``include_organization_memberships``: include private docs owned by any organization
+      the user belongs to (``owner_type=ORG`` and ``owner IN user.orgs``). Used by GraphQL
+      so organization members see private repos belonging to their orgs.
+
+    Flags are independent OR-combined extensions. Staff bypass goes through
+    ``apply_document_public_visibility_filter`` (this helper itself does not check staff).
+    """
+    criteria = Q('term', public_can_view=True)
+    if not getattr(user, 'is_authenticated', False):
+        return criteria
+
+    private_criteria = None
+    username = getattr(user, 'username', None)
+    if username and include_creator_private_access:
+        private_criteria = Q('term', created_by=username)
+
+    if username and include_owner_private_access:
+        owner_criteria = Q('term', owner_type=USER_OBJECT_TYPE) & Q('term', owner=username.lower())
+        private_criteria = owner_criteria if private_criteria is None else private_criteria | owner_criteria
+
+    if include_organization_memberships:
+        organization_mnemonics = [
+            mnemonic.lower() for mnemonic in user.organizations.values_list('mnemonic', flat=True)
+        ]
+        if organization_mnemonics:
+            org_criteria = Q('term', owner_type=ORG_OBJECT_TYPE) & Q('terms', owner=organization_mnemonics)
+            private_criteria = org_criteria if private_criteria is None else private_criteria | org_criteria
+
+    if private_criteria is None:
+        return criteria
+
+    return criteria | (Q('term', public_can_view=False) & private_criteria)
+
+
+def apply_document_public_visibility_filter(  # pylint: disable=too-many-arguments
+    search,
+    user,
+    include_creator_private_access=False,
+    include_owner_private_access=False,
+    include_organization_memberships=False,
+):
+    """Apply a shared Elasticsearch visibility filter without changing staff searches."""
+    if getattr(user, 'is_staff', False):
+        return search
+
+    return search.filter(
+        get_document_public_visibility_criteria(
+            user,
+            include_creator_private_access=include_creator_private_access,
+            include_owner_private_access=include_owner_private_access,
+            include_organization_memberships=include_organization_memberships,
+        )
+    )
 
 
 class CustomESFacetedSearch(FacetedSearch):

core/common/views.py

@@ -29,7 +29,7 @@
     CANONICAL_URL_REQUEST_PARAM, CHECKSUMS_PARAM, ACCESS_TYPE_NONE
 from core.common.exceptions import Http400
 from core.common.mixins import PathWalkerMixin
-from core.common.search import CustomESSearch
+from core.common.search import CustomESSearch, get_document_public_visibility_criteria
 from core.common.serializers import RootSerializer
 from core.common.swagger_parameters import all_resource_query_param
 from core.common.throttling import ThrottleUtil
@@ -704,7 +704,7 @@ def get_public_criteria(self):
             if self.document_model in [OrganizationDocument]:
                 criteria |= (Q('term', public_can_view=False) & Q('term', user=username))
             if self.is_concept_container_document_model() or self.is_source_child_document_model():
-                criteria |= (Q('term', public_can_view=False) & Q('term', created_by=username))
+                return get_document_public_visibility_criteria(user, include_creator_private_access=True)
 
         return criteria
 

core/concepts/documents.py

@@ -12,6 +12,11 @@ class Index:
         name = 'concepts'
         settings = {'number_of_shards': 1, 'number_of_replicas': 0}
 
+    # Preserve ORM semantics for direct GraphQL projections without changing REST search fields.
+    is_active = fields.BooleanField(attr='is_active')
+    is_head = fields.BooleanField()
+    display_name = fields.TextField(attr='display_name')
+
     id = fields.TextField(attr='mnemonic')
     id_lowercase = fields.KeywordField(attr='mnemonic', normalizer="lowercase")
     id_raw = fields.KeywordField(attr='mnemonic')
@@ -267,3 +272,8 @@ def get_mapped_codes(instance):
                 else:
                     other_mapped_codes.append(to_concept_code)
         return same_as_mapped_codes, other_mapped_codes, verbose_info
+
+    @staticmethod
+    def prepare_is_head(instance):
+        """Match the versioned-object predicate used by Source.get_concepts_queryset."""
+        return instance.id == instance.versioned_object_id

core/graphql/README.md

@@ -0,0 +1,123 @@
+# GraphQL concepts and source MVP
+
+This work carries the permission architecture reviewed in [PR #838](https://github.com/OpenConceptLab/oclapi2/pull/838)
+onto master `ab03e1c0`, and adds source metadata and selection-driven retrieval. The API version is `2.3.202-dev`.
+Master already contains the [PR #877](https://github.com/OpenConceptLab/oclapi2/pull/877) Strawberry bump:
+`strawberry-graphql==0.315.7`, with `strawberry-graphql-django==0.80.0`. These versions were retained.
+
+## Queries
+
+Open `/graphql/` for GraphiQL. Query arguments, returned fields, and summary fields include schema descriptions.
+Use the existing OCL token, OIDC bearer token, or session authentication. Invalid credentials are rejected before
+resolvers run; authenticated users still require the existing `graphql_api` group. Anonymous queries see public data.
+
+```graphql
+query Dictionary($org: String!, $source: String!, $version: String) {
+  source(org: $org, source: $source, version: $version) {
+    name
+    description
+    canonicalUrl
+    uri
+    classes
+    datatypes
+    mapTypes
+    externalSources { name uri }
+    summary { activeConcepts mappings }
+  }
+}
+```
+
+Variables: `{"org":"CIEL","source":"CIEL"}`. For personal repositories, replace `org` with `owner` (username).
+`uri` is the stored OCL relative URI, for example `/orgs/CIEL/sources/CIEL/`; releases include their version.
+The canonical field is spelled `canonicalUrl`.
+
+```graphql
+query FindConcepts($org: String, $source: String, $query: String!, $page: Int, $limit: Int) {
+  concepts(org: $org, source: $source, query: $query, page: $page, limit: $limit) {
+    totalCount
+    hasNextPage
+    versionResolved
+    results { conceptId display description conceptClass datatype { name } }
+  }
+}
+```
+
+Variables: `{"org":"CIEL","source":"CIEL","query":"hypertension","page":1,"limit":20}`.
+Omit both `org` and `source` for global search. `conceptIds` performs exact, case-sensitive mnemonic matching,
+deduplicates the input, and preserves its order; it takes precedence over `query`. Supply `page` and `limit`
+together; the supported result window is 10,000. Without pagination, index responses are capped at 10,000;
+`totalCount` remains the total number of matches. Omitted versions use HEAD, falling back to the latest released
+version only if HEAD is absent; explicit missing versions do not fall back.
+
+## Data access and permissions
+
+| Selected payload | Retrieval |
+| --- | --- |
+| Source `name`, `canonicalUrl`, `uri` | Source index projection, including source/version resolution. `uri` is rebuilt from owner, owner type, mnemonic and version rather than stored |
+| Concept `id`, `conceptId`, `externalId`, `display`, `conceptClass`, `datatype { name }` | Concept index projection; no ORM concept hydration |
+| Source `description`, concept `description` | Not indexed; selecting either routes that request through the ORM |
+| Only concept counts/pagination metadata | Elasticsearch request with zero result hits |
+| Concept names, mappings, extras, audit metadata, datatype details | ORM hydration with selected concept columns and relations |
+| Source classes, datatypes, map types, external sources, summary | Existing version-scoped model querysets; only selected aggregates execute |
+
+Aliases, fragments, `@skip`, `@include`, and nested `__typename` selections participate in planning.
+Elasticsearch `_source` is restricted to selected fields. An empty successful direct projection is authoritative;
+it does not trigger a database scan. Expected index/transport failures fall back to permission-checked ORM queries.
+The older hydrated text-search path retains its empty-index database fallback.
+
+Counts and distinct labels use active, non-retired records. `summary.mappings` counts active, non-retired mappings.
+`externalSources` is the deduplicated set of outbound target repositories, excluding the current source and linked
+private targets the caller cannot view. Unresolved external URIs are taken from visible mappings.
+
+Repository permission checks reuse the shared REST visibility rule directly, without fabricated requests.
+Both owner mnemonic and owner type scope index lookups. Concept visibility relies on the indexed
+`public_can_view` flag that `core/sources/signals.py` already propagates from the parent repository, and
+mapping hydration independently checks target visibility. HEAD uses the same versioned-object identity as
+`Source.get_concepts_queryset()`, while releases use their membership lists.
+
+SQL-free data retrieval does not mean SQL-free authentication: session/token lookup and organization membership
+resolution can query the database. Tests verify zero SQL for anonymous public index projections. As with the
+existing REST index, indexed results reflect Elasticsearch refresh and indexing propagation latency.
+
+## Rollout
+
+No database migrations or new environment variables are introduced. Refresh the source and concept indexes
+before serving this GraphQL version: older concept documents lack the `is_active`, `is_head` and `display_name`
+projection fields, and older source documents lack `is_active`. Do not use incomplete indexes during the
+rollout; concept projections filter on `is_active` and `is_head`, so an unrefreshed index returns zero
+concepts without raising an error.
+
+Use the existing indexing procedure to apply the additive mappings and repopulate both models. For a deployment
+that recreates indexes, use its established rebuild procedure; do not rebuild live indexes without accounting for
+REST search availability. A full population command for the existing application container is:
+
+```sh
+docker exec oclapi2-api-1 python manage.py search_index --populate --models sources.Source concepts.Concept -f --parallel
+```
+
+Source permission/activity propagation refreshes the corresponding concept projection flags. Existing
+REST search relevance and excluded-word semantics are preserved; unrelated search refactors from PR #838 were
+not carried over. Its corrected permission sharing and documented Strawberry auth extension were retained.
+
+## Verification
+
+```sh
+docker exec oclapi2-api-1 python manage.py test core.graphql.tests --keepdb --noinput -v2
+docker exec oclapi2-api-1 pylint -j2 core/graphql core/common/permissions.py core/common/search.py core/common/views.py core/sources/signals.py core/integration_tests/test_graphql_projection.py
+```
+
+`core.integration_tests.test_graphql_projection` requires `settings.ES_ENABLED=True`. It creates uniquely named
+indexes and removes them after each test. Run it only against a test Elasticsearch service: shared fixture setup
+can also exercise normal indexing hooks. It covers real index preparation, zero SQL, owner isolation, HEAD/release
+selection, inactive/retired filtering, private-repository visibility, the rebuilt source URI, and the
+database fallback for `description`.
+
+For this worktree, verification used a copy at `/tmp/graphql-sources-20260906` inside the existing API container,
+the dedicated database `test_graphql_sources_20260906`, and a temporary Elasticsearch container. The running app's
+`/code` checkout and search indexes were not changed. Coverage uses a temporary runner that selects Python's YAML
+loader because the container's C YAML loader fails under coverage instrumentation; application dependencies were
+not modified to work around that test-environment issue.
+
+Verified results: **75 distinct tests passed**, including six tests against real Elasticsearch and the focused
+REST/source-signal regressions. Coverage of `core.graphql` (excluding test files) is **98%**: 739 of 753 statements.
+Pylint completed without findings. No changes were made to the application's installed dependency versions.

core/graphql/constants.py

@@ -0,0 +1,56 @@
+"""Shared GraphQL error metadata used by views, resolvers, and tests."""
+
+from typing import Optional
+
+from strawberry.exceptions import GraphQLError
+
+AUTHENTICATION_FAILED = 'AUTHENTICATION_FAILED'
+FORBIDDEN = 'FORBIDDEN'
+SEARCH_UNAVAILABLE = 'SEARCH_UNAVAILABLE'
+VALIDATION_ERROR = 'VALIDATION_ERROR'
+
+GRAPHQL_ERROR_DEFINITIONS = {
+    AUTHENTICATION_FAILED: {
+        'message': 'Authentication failure',
+        'description': 'The provided credentials are invalid for the GraphQL API.',
+    },
+    FORBIDDEN: {
+        'message': 'Forbidden',
+        'description': 'The current user cannot access the requested repository.',
+    },
+    SEARCH_UNAVAILABLE: {
+        'message': 'Search unavailable',
+        'description': 'Global concept search requires Elasticsearch and is temporarily unavailable.',
+    },
+    VALIDATION_ERROR: {
+        'message': 'Validation error',
+        'description': 'Client supplied arguments that violate input validation rules.',
+    },
+}
+EXPECTED_GRAPHQL_ERROR_CODES = frozenset(GRAPHQL_ERROR_DEFINITIONS.keys())
+
+
+def build_expected_graphql_error(code, message: Optional[str] = None):
+    """Return a GraphQL error with a stable code and a short client-facing description.
+
+    Pass ``message`` to override the default human-readable message while preserving
+    the machine-readable ``code``.
+    """
+    detail = GRAPHQL_ERROR_DEFINITIONS[code]
+    return GraphQLError(
+        message or detail['message'],
+        extensions={
+            'code': code,
+            'description': detail['description'],
+        },
+    )
+
+
+def build_validation_error(message: str):
+    """Shortcut for client-side validation failures that should not be logged as server errors."""
+    return build_expected_graphql_error(VALIDATION_ERROR, message=message)
+
+
+def get_graphql_error_code(error):
+    """Read the machine-readable error code attached to a GraphQL error when present."""
+    return (getattr(error, 'extensions', None) or {}).get('code')

core/graphql/extensions.py

@@ -0,0 +1,21 @@
+"""Strawberry schema extensions used to enforce cross-cutting GraphQL policies."""
+
+from typing import Iterator
+
+from graphql import ExecutionResult
+from strawberry.extensions import SchemaExtension
+
+from .constants import AUTHENTICATION_FAILED, build_expected_graphql_error
+
+
+class AuthStatusExtension(SchemaExtension):
+    """Reject requests with invalid credentials before any resolver runs."""
+
+    def on_execute(self) -> Iterator[None]:
+        context = self.execution_context.context
+        if getattr(context, 'auth_status', 'none') == 'invalid':
+            self.execution_context.result = ExecutionResult(
+                data=None,
+                errors=[build_expected_graphql_error(AUTHENTICATION_FAILED)],
+            )
+        yield