queueserver: port the tiled-v0.2.12 auth stack (upstream PR #81) + three upstream fixes

.gitignore

@@ -16,3 +16,6 @@ htmlcov/
 
 # OS
 .DS_Store
+
+# reflow2 design graph (local only, never pushed)
+.reflow2/

backend/queueserver_service/bluesky-httpserver/bluesky_httpserver/__init__.py

@@ -18,7 +18,7 @@
 
 Every upstream ``bluesky_httpserver`` submodule maps 1:1 onto
 ``queueserver_service.http`` (``server``, ``config``, ``settings``,
-``authentication``, ``authenticators``, ``authorization``, ``core``,
+``authentication``, ``authenticators``, ``authorization``, ``protocols``, ``core``,
 ``schemas``, ``resources``, ``console_output``, ``utils``, ``app``, ``routers``,
 ``database``, ``config_schemas``). Rather than eagerly importing all of them --
 which would pull the entire FastAPI application stack on a bare

backend/queueserver_service/docs/source/http/configuration.rst

@@ -294,6 +294,93 @@ See the documentation on ``LDAPAuthenticator`` for more details.
 
     authenticators.LDAPAuthenticator
 
+OIDC Authenticator
+++++++++++++++++++
+
+``OIDCAuthenticator`` integrates the server with third-party OpenID Connect providers
+such as Google, Microsoft Entra ID, ORCID and others. The server does not process user
+passwords directly: authentication is delegated to the provider and the server validates
+the returned OIDC token.
+
+General setup steps:
+
+#. Register an application with the OIDC provider.
+#. Configure redirect URIs for the provider application. For provider name ``entra`` and
+   host ``https://your-server.example`` the redirect URIs are:
+
+   - ``https://your-server.example/api/auth/provider/entra/code``
+   - ``https://your-server.example/api/auth/provider/entra/device_code``
+
+#. Store the client secret in environment variable and reference it in config.
+#. Use provider's ``.well-known/openid-configuration`` URL.
+
+Typical ``well_known_uri`` values:
+
+- Google: ``https://accounts.google.com/.well-known/openid-configuration``
+- Microsoft Entra ID: ``https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration``
+- ORCID: ``https://orcid.org/.well-known/openid-configuration``
+
+
+Example configuration (Google)::
+
+    authentication:
+      providers:
+        - provider: google
+          authenticator: queueserver_service.http.authenticators:OIDCAuthenticator
+          args:
+            audience: <google-client-id>
+            client_id: <google-client-id>
+            client_secret: ${BSKY_GOOGLE_SECRET}
+            well_known_uri: https://accounts.google.com/.well-known/openid-configuration
+
+.. note::
+
+    The name used in ``api_access/args/users`` must match the identity string produced by
+    the authenticator for your provider configuration. Verify with ``/api/auth/whoami`` after
+    successful login.
+
+See the documentation on ``OIDCAuthenticator`` for parameter details.
+
+.. autosummary::
+   :nosignatures:
+   :toctree: generated
+
+    authenticators.OIDCAuthenticator
+
+ENTRA Authenticator
++++++++++++++++++++
+
+``EntraAuthenticator`` inherits from the ``ProxiedOIDCAuthenticator`` and provides
+additional ENTRA/MS specific ways to determine the actual username, while still
+using the OIDC workflow. It will by default attempt to extract a human-readable
+username from the claims in the OIDC token. Alternatively a graph parameter
+can be specified, at which point after ENTRA returns a valid login and identity
+a GraphAPI call is made to request the provided parameter, which is then used
+in place of any claim as the username. This later method is the method recommended
+by MS.
+
+
+Example configuration (Microsoft Entra ID)::
+
+    authentication:
+      providers:
+        - provider: entra
+          authenticator: queueserver_service.http.authenticators:EntraAuthenticator
+          args:
+            audience: 00000000-0000-0000-0000-000000000000
+            client_id: 00000000-0000-0000-0000-000000000000
+            device_flow_client_id: 00000000-0000-0000-0000-000000000000
+            client_secret: ${BSKY_ENTRA_SECRET}
+            well_known_uri: https://login.microsoftonline.com/<tenant-id>/v2.0/.well-known/openid-configuration
+            confirmation_message: "You have logged in successfully."
+            extra_scopes: 'User.Read'
+            graph_username_attribute: "some_graph_param"
+
+.. autosummary::
+   :nosignatures:
+   :toctree: generated
+
+    authenticators.EntraAuthenticator
 
 Expiration Time for Tokens and Sessions
 +++++++++++++++++++++++++++++++++++++++

backend/queueserver_service/docs/source/http/usage.rst

@@ -154,6 +154,61 @@ Then users ``bob``, ``alice`` and ``tom`` can log into the server as ::
 
 If authentication is successful, then the server returns access and refresh tokens.
 
+Logging in with OIDC Providers (Google, Entra, ORCID, ...)
+-----------------------------------------------------------
+
+For providers configured with ``OIDCAuthenticator``, use provider-specific endpoints
+under ``/api/auth/provider/<provider-name>/...``.
+
+Browser-first flow
+******************
+
+If you are already in a browser context, open:
+
+``<hostname>/api/auth/provider/<provider-name>/authorize``
+
+This redirects to the OIDC provider login page and then back to the server callback.
+
+This can similarly be achieved using ``httpie`` by opening the URL in a browser after getting
+the authorization URI from the server::
+
+  http POST http://localhost:60610/api/auth/provider/entra/authorize
+
+Which will return a token back to the bluesky http server after the user logs in to the provider
+in their browser (or automatically if already logged in). The user then gets a token
+for the bluesky HTTP server to use for subsequent API requests. This flow can be used
+even when using the bluesky queueserver api in a terminal so long as that session can
+spawn a browser for the user to log in to the provider.
+
+CLI/device flow
+***************
+
+For terminal clients (i.e. no browser possible), start with
+``POST /api/auth/provider/<provider-name>/authorize``.
+The response includes:
+
+- ``authorization_uri``: open this URL in a browser
+- ``verification_uri``: polling endpoint for the terminal client
+- ``device_code`` and ``interval``: values for polling
+
+Example using ``httpie`` (provider ``entra``)::
+
+  http POST http://localhost:60610/api/auth/provider/entra/authorize
+
+After opening ``authorization_uri`` in a browser and completing provider login,
+poll ``verification_uri`` using ``device_code`` until tokens are issued::
+
+  http POST http://localhost:60610/api/auth/provider/entra/token \
+    device_code='<device_code_from_authorize_response>'
+
+When authorization is still pending, the endpoint returns ``authorization_pending``.
+When complete, it returns access and refresh tokens.
+
+.. note::
+
+    In common same-device flows the callback can complete automatically without manually
+    typing the user code. Manual code entry remains available as a fallback path.
+
 Generating API Keys
 -------------------
 

backend/queueserver_service/pyproject.toml

@@ -59,6 +59,9 @@ dependencies = [
     # missing from httpserver's own requirements (upstream omission).
     "alembic",
     "bluesky-queueserver-api",
+    # cachetools: TTL cache on the OIDC JWKS key fetch (authenticators.py,
+    # tiled-v0.2.12 alignment).
+    "cachetools",
     "fastapi",
     "httpx",
     "ldap3",
@@ -179,3 +182,16 @@ force-exclude = '''
   )/
 )
 '''
+
+[dependency-groups]
+dev = [
+    "aiosqlite>=0.22.1",
+    "cryptography>=50.0.0",
+    "h5py>=3.16.0",
+    "happi>=3.0.1",
+    "matplotlib>=3.11.0",
+    "pandas>=3.0.3",
+    "pytest-asyncio>=1.4.0",
+    "pytest-xprocess>=1.0.2",
+    "respx>=0.23.1",
+]

backend/queueserver_service/queueserver_service/http/app.py

@@ -14,11 +14,12 @@
 from fastapi import APIRouter, FastAPI, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
 
-from .authentication import Mode
+from .authenticators import ProxiedOIDCAuthenticator
 from .console_output import CollectPublishedConsoleOutput, ConsoleOutputStream, SystemInfoStream
 from .core import PatchedStreamingResponse
 from .database.core import purge_expired
 from .openapi_config import custom_openapi
+from .protocols import ExternalAuthenticator, InternalAuthenticator
 from .resources import SERVER_RESOURCES as SR
 from .routers import (
     admin as admin_router,
@@ -159,9 +160,9 @@ def build_app(authentication=None, api_access=None, resource_access=None, server
         logger.info("All custom routers are included successfully.")
 
     from .authentication import (
+        add_external_routes,
+        add_internal_routes,
         base_authentication_router,
-        build_auth_code_route,
-        build_handle_credentials_route,
         oauth2_scheme,
     )
 
@@ -175,44 +176,21 @@ def build_app(authentication=None, api_access=None, resource_access=None, server
         first_provider = authentication["providers"][0]["provider"]
         oauth2_scheme.model.flows.password.tokenUrl = f"/api/auth/provider/{first_provider}/token"
         # Authenticators provide Router(s) for their particular flow.
-        # Collect them in the authentication_router.
-
+        # Collect them in the authentication_router. The authenticator's
+        # class (InternalAuthenticator vs ExternalAuthenticator protocol)
+        # determines the routes it gets — the old per-instance `mode` flag
+        # is gone (upstream PR #81 / tiled v0.2.12 alignment).
         for spec in authentication["providers"]:
             provider = spec["provider"]
             authenticator = spec["authenticator"]
-            mode = authenticator.mode
-            if mode == Mode.password:
-                authentication_router.post(
-                    f"/provider/{provider}/token",
-                    summary=f"Exchange username+password for tokens ({provider})",
-                    description=(
-                        f"OAuth2 password-flow token endpoint for the `{provider}` "
-                        "authenticator. Form fields: `username`, `password`. Returns "
-                        "access + refresh tokens."
-                    ),
-                    tags=["Auth"],
-                )(build_handle_credentials_route(authenticator, provider))
-            elif mode == Mode.external:
-                auth_code_summary = f"Exchange an external-identity callback for a refresh token ({provider})"
-                auth_code_description = (
-                    f"External-identity auth-code endpoint for the `{provider}` authenticator. "
-                    "Accepts the callback from the upstream IdP (OIDC / LDAP / SAML) and "
-                    "returns a refresh token the client can use to obtain access tokens."
-                )
-                authentication_router.get(
-                    f"/provider/{provider}/code",
-                    summary=auth_code_summary,
-                    description=auth_code_description,
-                    tags=["Auth"],
-                )(build_auth_code_route(authenticator, provider))
-                authentication_router.post(
-                    f"/provider/{provider}/code",
-                    summary=auth_code_summary,
-                    description=auth_code_description,
-                    tags=["Auth"],
-                )(build_auth_code_route(authenticator, provider))
+            if isinstance(authenticator, InternalAuthenticator):
+                add_internal_routes(authentication_router, provider, authenticator)
+            elif isinstance(authenticator, ExternalAuthenticator):
+                add_external_routes(authentication_router, provider, authenticator)
+                if isinstance(authenticator, ProxiedOIDCAuthenticator):
+                    app.state.provider = provider
             else:
-                raise ValueError(f"unknown authentication mode {mode}")
+                raise ValueError(f"unknown authenticator type {type(authenticator)}")
             for custom_router in getattr(authenticator, "include_routers", []):
                 authentication_router.include_router(custom_router, prefix=f"/provider/{provider}")
 
@@ -256,9 +234,11 @@ async def startup_event():
             from .database import orm
             from .database.core import (  # make_admin_by_identity,
                 REQUIRED_REVISION,
+                DatabaseUpgradeNeeded,
                 UninitializedDatabase,
                 check_database,
                 initialize_database,
+                upgrade,
             )
 
             connect_args = {}
@@ -276,6 +256,10 @@ async def startup_event():
                 )
                 initialize_database(engine)
                 logger.info("Database initialized.")
+            except DatabaseUpgradeNeeded:
+                logger.info(f"Database at {redacted_url} is out of date. Upgrading to {REQUIRED_REVISION}...")
+                upgrade(engine, REQUIRED_REVISION)
+                logger.info("Database upgraded.")
             else:
                 logger.info(f"Connected to existing database at {redacted_url}.")
             # Identity-based admin designation (qserver_admins/tiled_admins) is
@@ -423,10 +407,30 @@ async def purge_expired_sessions_and_api_keys():
 
     @app.on_event("shutdown")
     async def shutdown_event():
-        await SR.RM.close()
-        await SR.console_output_loader.stop()
-        await SR.console_output_stream.stop()
-        await SR.system_info_stream.stop()
+        """Safely shutdown and perform the cleanup robustly
+
+        This change ensures that the application shuts down and cleans up resources even if there is
+        a problem, without silencing the errors.
+        """
+        for task in getattr(app.state, "tasks", []):
+            task.cancel()
+        for closer_name in (
+            "console_output_loader",
+            "console_output_stream",
+            "system_info_stream",
+        ):
+            closer = getattr(SR, closer_name, None)
+            if closer is not None:
+                try:
+                    await closer.stop()
+                except Exception:
+                    logger.exception("Error stopping %s", closer_name)
+        rm = getattr(SR, "RM", None)
+        if rm is not None:
+            try:
+                await rm.close()
+            except Exception:
+                logger.exception("Error closing REManagerAPI connection")
 
     @lru_cache(1)
     def override_get_authenticators():