From a64354c9b0de91e1a2199b0992e63973a69c8b10 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Mon, 29 Jun 2026 21:31:29 -0700 Subject: [PATCH 01/23] [Feature] Add PPL `rest` command (Calcite system row source) Add a leading `rest ` command that exposes a curated, read-only, fixed-schema set of in-cluster management endpoints as a PPL table, modeled as a system row source bridged through visitRelation (the same seam as describe and the system-index family), so it runs on the Calcite engine without the unsupported table-function path. - Grammar/AST: REST/TIMEOUT tokens, restCommand rule, RestRelation, AstBuilder visit, query anonymizer. - Execution: RestSourceTable -> CalciteLogicalRestScan / CalciteEnumerableRestScan; RestEndpointRegistry (read-only allow-list + fixed schema + accepted args); RestEnumerator/RestRequest dispatch via OpenSearchClient (NodeClient in-cluster, RestClient standalone). - 9 endpoints: cluster health/state/settings, cat indices/nodes/cluster_manager/ plugins/shards, resolve/index. - Output shaping: numeric type normalization, id-to-name resolution, role-name expansion, structural flattening, graceful null degrade. - Args: count caps emitted rows; timeout reserved but rejected with 400; get-args applied server-side with per-arg value validation (local on health, health on cat/indices, expand_wildcards on resolve/index). Undeclared arg or out-of-domain value is rejected with a 400. level and include_defaults are deferred to a later release; flat_settings is dropped as redundant. - Error handling: blank endpoint, negative count, disallowed arg, and uncoercible value all surface clean 400s rather than 500s. Tests: CalcitePPLRestIT 25/25, RestEndpointRegistryTest 16, RestSourceTableTest 10. Signed-off-by: Louis Chu --- .../opensearch/sql/ast/tree/RestRelation.java | 25 ++ .../sql/utils/SystemIndexUtils.java | 106 +++++ docs/user/ppl/cmd/rest.md | 62 +++ docs/user/ppl/index.md | 1 + .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 9 + .../sql/calcite/remote/CalcitePPLRestIT.java | 213 +++++++++ .../sql/ppl/NewAddedCommandsIT.java | 13 + .../src/main/antlr4/OpenSearchPPLLexer.g4 | 2 + .../src/main/antlr4/OpenSearchPPLParser.g4 | 11 + .../opensearch/client/OpenSearchClient.java | 101 +++++ .../client/OpenSearchNodeClient.java | 252 +++++++++++ .../client/OpenSearchRestClient.java | 266 ++++++++++++ .../planner/rules/EnumerableRestScanRule.java | 46 ++ .../planner/rules/OpenSearchIndexRules.java | 2 + .../storage/OpenSearchStorageEngine.java | 7 +- .../storage/rest/AbstractCalciteRestScan.java | 41 ++ .../rest/CalciteEnumerableRestScan.java | 89 ++++ .../storage/rest/CalciteLogicalRestScan.java | 48 +++ .../storage/rest/RestEndpointRegistry.java | 407 ++++++++++++++++++ .../storage/rest/RestEnumerator.java | 86 ++++ .../opensearch/storage/rest/RestRequest.java | 47 ++ .../storage/rest/RestSourceTable.java | 80 ++++ .../rest/RestEndpointRegistryTest.java | 213 +++++++++ .../storage/rest/RestSourceTableTest.java | 135 ++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 12 + .../opensearch/sql/ppl/parser/AstBuilder.java | 33 ++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 19 + .../sql/ppl/calcite/CalcitePPLRestTest.java | 66 +++ .../sql/ppl/parser/AstBuilderTest.java | 28 ++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 12 + 32 files changed, 2434 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java create mode 100644 docs/user/ppl/cmd/rest.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..200a9f19681 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,9 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +37,109 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + StringBuilder h = new StringBuilder(); + for (byte b : s.getBytes(StandardCharsets.UTF_8)) { + h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return h.toString(); + } + + private static String fromHex(String h) { + byte[] bytes = new byte[h.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = + (byte) + ((Character.digit(h.charAt(2 * i), 16) << 4) + + Character.digit(h.charAt(2 * i + 1), 16)); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md new file mode 100644 index 00000000000..11f35e2ee86 --- /dev/null +++ b/docs/user/ppl/cmd/rest.md @@ -0,0 +1,62 @@ +# rest + +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. + +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. + +## Syntax + +The `rest` command has the following syntax: + +```syntax +rest [count=] [timeout=] [= ...] +``` + +## Parameters + +The `rest` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | +| `count=` | Optional | Caps the number of emitted rows. | +| `timeout=` | Optional | Request timeout passed to the transport action, for example `30s`. | +| `=` | Optional | Endpoint query arguments, validated per endpoint (for example `level=indices` for `/_cluster/health`). | + +## Allow-list + +`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. + +| Endpoint | Output columns | +| --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | + +## Example 1: Reading cluster health + +The following query reads cluster health and projects two columns: + +```ppl +| rest /_cluster/health | fields status, number_of_nodes +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++--------+-----------------+ +| status | number_of_nodes | +|--------+-----------------| +| green | 1 | ++--------+-----------------+ +``` + +## Example 2: Listing indexes from cat indices + +The following query lists indexes and filters and sorts them on the Calcite plan: + +```ppl +| rest /_cat/indices | where health = "green" | sort index | fields index, health, pri +``` + +The downstream `where`, `sort`, and `fields` compose over the `rest` row source exactly like any index scan. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 37947113800..939684c0ecc 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,6 +74,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index d46649d56d9..da434750a96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -57,6 +57,7 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, + CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 95c47b9b0b7..784e324bfbd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -64,6 +64,15 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. + @Test + public void explainRestCommand() throws IOException { + String result = explainQueryToString("| rest \"/_cluster/health\" | fields status"); + Assert.assertTrue( + "Expected a rest scan node in the explain output, got: " + result, + result.contains("RestScan") || result.contains("rest")); + } + @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..c2ea6069996 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,213 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also verifies that a non-allow-listed / mutating endpoint is refused. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = + executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + } + + @Test + public void testRestClusterHealthDataRows() throws IOException { + // Single-node test cluster: exactly one node, status is green or yellow. + JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() throws IOException { + assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() throws IOException { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + } + + @Test + public void testRestRejectsTimeoutArg() throws IOException { + assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. The negative-case check for allow-list and secret-filter enforcement. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals( + 400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } + + @Test + public void testRestCatIndicesSchema() throws IOException { + // Schema is fixed by the registry, independent of how many indices exist. + JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); + verifySchema(result, schema("index", "string"), schema("health", "string")); + } + + @Test + public void testRestCatIndicesReturnsCreatedIndex() throws IOException { + // Create a known index, then confirm rest surfaces it and downstream where/fields compose. + client().performRequest(new Request("PUT", "/rest_cat_test")); + JSONObject result = + executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); + verifyDataRows(result, rows("rest_cat_test")); + } + + @Test + public void testRestCatNodesSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); + verifySchema(result, schema("name", "string"), schema("cpu", "int")); + } + + @Test + public void testRestCatNodesSingleNode() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatClusterManagerSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); + verifySchema(result, schema("node", "string"), schema("id", "string")); + } + + @Test + public void testRestCatClusterManagerSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatPluginsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); + verifySchema(result, schema("component", "string"), schema("version", "string")); + } + + @Test + public void testRestCatShardsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); + verifySchema( + result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); + } + + @Test + public void testRestClusterStateSchema() throws IOException { + // Assert the string columns; version is the LONG epoch column. + JSONObject result = + executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); + verifySchema( + result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + } + + @Test + public void testRestClusterStateSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterSettingsSchema() throws IOException { + // Schema is registry-fixed regardless of how many settings are configured. + JSONObject result = + executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + verifySchema( + result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); + } + + @Test + public void testRestResolveIndexSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { + client().performRequest(new Request("PUT", "/rest_resolve_test")); + JSONObject result = + executeQuery( + "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); + verifyDataRows(result, rows("rest_resolve_test", "index")); + } + + // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { + // health filters rows server-side; a healthy cluster has no red indices, so count is 0. + JSONObject result = + executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + verifyDataRows(result, rows(0)); + } + + @Test + public void testRestResolveIndexExpandWildcardsArg() throws IOException { + // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. + JSONObject result = + executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestRejectsDroppedLevelArg() throws IOException { + // level was dropped from the allow-list (no-op against the fixed health schema). + assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + } + + @Test + public void testRestRejectsBadArgValue() throws IOException { + assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..8d5912ab199 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,6 +32,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index 2248374d8d9..8ba747614da 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index 451824d6992..1d13e8af046 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -62,6 +62,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | AD | ML @@ -113,6 +114,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; explainCommand : EXPLAIN explainMode ; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 68350c5a0fd..3b9c3619521 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,4 +114,105 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); + + /** + * Read-only cluster health snapshot for the {@code rest} command (backs {@code + * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's + * security thread-context; performs no privilege escalation and mutates nothing. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of health field name to value + */ + default Map clusterHealth(Map params) { + throw new UnsupportedOperationException("clusterHealth is not supported by this client"); + } + + /** + * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One + * map per index. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per index + */ + default List> catIndices(Map params) { + throw new UnsupportedOperationException("catIndices is not supported by this client"); + } + + /** + * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map + * per node with resource state. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per node + */ + default List> catNodes(Map params) { + throw new UnsupportedOperationException("catNodes is not supported by this client"); + } + + /** + * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code + * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map describing the cluster manager node + */ + default List> catClusterManager(Map params) { + throw new UnsupportedOperationException("catClusterManager is not supported by this client"); + } + + /** + * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One + * map per installed plugin per node. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per plugin + */ + default List> catPlugins(Map params) { + throw new UnsupportedOperationException("catPlugins is not supported by this client"); + } + + /** + * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map + * per shard. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per shard + */ + default List> catShards(Map params) { + throw new UnsupportedOperationException("catShards is not supported by this client"); + } + + /** + * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code + * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, + * cluster_manager_node). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of cluster-state field name to value + */ + default Map clusterState(Map params) { + throw new UnsupportedOperationException("clusterState is not supported by this client"); + } + + /** + * Read-only cluster-settings listing for the {@code rest} command (backs {@code + * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per setting + */ + default List> clusterSettings(Map params) { + throw new UnsupportedOperationException("clusterSettings is not supported by this client"); + } + + /** + * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). + * One map per resolved index, alias, or data stream (name, type). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per resolved name + */ + default List> resolveIndex(Map params) { + throw new UnsupportedOperationException("resolveIndex is not supported by this client"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..9ce9ff79ea8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -25,6 +27,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -285,4 +288,253 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + return flattenHealth(response); + } + + @Override + public List> catIndices(Map params) { + ClusterHealthResponse response = + client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new java.util.LinkedHashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } + + @Override + public List> catNodes(Map params) { + org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = + new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); + statsRequest.all(); + org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = + client.admin().cluster().nodesStats(statsRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { + org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); + Map row = new java.util.LinkedHashMap<>(); + row.put("name", node.getName()); + row.put("ip", node.getHostAddress()); + row.put( + "node_role", + node.getRoles().stream() + .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) + .sorted() + .collect(java.util.stream.Collectors.joining(","))); + row.put( + "heap_percent", + ns.getJvm() == null || ns.getJvm().getMem() == null + ? null + : (int) ns.getJvm().getMem().getHeapUsedPercent()); + row.put( + "ram_percent", + ns.getOs() == null || ns.getOs().getMem() == null + ? null + : (int) ns.getOs().getMem().getUsedPercent()); + row.put( + "cpu", + ns.getProcess() == null || ns.getProcess().getCpu() == null + ? null + : (int) ns.getProcess().getCpu().getPercent()); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + List> rows = new java.util.ArrayList<>(); + if (cm != null) { + Map row = new java.util.LinkedHashMap<>(); + row.put("id", cm.getId()); + row.put("host", cm.getHostName()); + row.put("ip", cm.getHostAddress()); + row.put("node", cm.getName()); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = + new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); + infoRequest.all(); + org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = + client.admin().cluster().nodesInfo(infoRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { + org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = + info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); + if (plugins == null) { + continue; + } + for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", info.getNode().getName()); + row.put("component", pi.getName()); + row.put("version", pi.getVersion()); + rows.add(row); + } + } + return rows; + } + + @Override + public List> catShards(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.cluster.routing.ShardRouting sr : + response.getState().getRoutingTable().allShards()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("index", sr.getIndexName()); + row.put("shard", sr.id()); + row.put("prirep", sr.primary() ? "p" : "r"); + row.put("state", sr.state().name()); + org.opensearch.cluster.node.DiscoveryNode n = + sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); + row.put("node", n == null ? null : n.getName()); + rows.add(row); + } + return rows; + } + + @Override + public Map clusterState(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName().value()); + row.put("state_uuid", response.getState().stateUUID()); + row.put("version", response.getState().version()); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + row.put("cluster_manager_node", cm == null ? null : cm.getName()); + return row; + } + + @Override + public List> clusterSettings(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + collectSettings(response.getState().metadata().persistentSettings(), "persistent", rows); + collectSettings(response.getState().metadata().transientSettings(), "transient", rows); + return rows; + } + + private void collectSettings( + org.opensearch.common.settings.Settings settings, + String tier, + List> rows) { + if (settings == null) { + return; + } + for (String key : settings.keySet()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("setting", key); + row.put("value", settings.get(key)); + row.put("tier", tier); + rows.add(row); + } + } + + @Override + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = + expandWildcards == null + ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}) + : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}, + org.opensearch.action.support.IndicesOptions.fromParameters( + expandWildcards, + null, + null, + null, + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request + .DEFAULT_INDICES_OPTIONS)); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = + client + .execute(org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : + response.getIndices()) { + rows.add(resolveRow(idx.getName(), "index")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : + response.getAliases()) { + rows.add(resolveRow(alias.getName(), "alias")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : + response.getDataStreams()) { + rows.add(resolveRow(ds.getName(), "data_stream")); + } + return rows; + } + + private Map resolveRow(String name, String type) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + return row; + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f369c0003b8..c9290408229 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,15 +8,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -28,6 +32,7 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -272,4 +277,265 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + try { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = + client.cluster().health(request, RequestOptions.DEFAULT); + return flattenHealth(response); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster health", e); + } + } + + @Override + public List> catIndices(Map params) { + try { + ClusterHealthResponse response = + client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + List> rows = new ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new HashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } catch (IOException e) { + throw new IllegalStateException("Failed to get cat indices", e); + } + } + + @Override + public List> catNodes(Map params) { + List> raw = + catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("ip", r.get("ip")); + row.put("node_role", r.get("node.role")); + row.put("heap_percent", asInt(r.get("heap.percent"))); + row.put("ram_percent", asInt(r.get("ram.percent"))); + row.put("cpu", asInt(r.get("cpu"))); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("id", r.get("id")); + row.put("host", r.get("host")); + row.put("ip", r.get("ip")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + List> raw = catJson("/_cat/plugins", "name,component,version"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("component", r.get("component")); + row.put("version", r.get("version")); + rows.add(row); + } + return rows; + } + + @Override + public List> catShards(Map params) { + List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("index", r.get("index")); + row.put("shard", asInt(r.get("shard"))); + row.put("prirep", r.get("prirep")); + row.put("state", r.get("state")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ + @SuppressWarnings("unchecked") + private List> catJson(String path, String columns) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + request.addParameter("format", "json"); + request.addParameter("h", columns); + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + List list = parser.list(); + List> rows = new ArrayList<>(); + for (Object o : list) { + rows.add((Map) o); + } + return rows; + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Integer asInt(Object value) { + if (value == null) { + return null; + } + try { + return (int) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public Map clusterState(Map params) { + Map state = + getJsonMap( + "/_cluster/state/master_node,version,metadata,nodes", + Map.of("filter_path", "cluster_name,state_uuid,version,cluster_manager_node,nodes")); + Map row = new HashMap<>(); + row.put("cluster_name", state.get("cluster_name")); + row.put("state_uuid", state.get("state_uuid")); + row.put("version", asLong(state.get("version"))); + Object cmId = state.get("cluster_manager_node"); + String cmName = null; + Object nodes = state.get("nodes"); + if (cmId != null && nodes instanceof Map) { + Object n = ((Map) nodes).get(cmId.toString()); + if (n instanceof Map) { + Object name = ((Map) n).get("name"); + cmName = name == null ? null : name.toString(); + } + } + row.put("cluster_manager_node", cmName); + return row; + } + + @Override + @SuppressWarnings("unchecked") + public List> clusterSettings(Map params) { + Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); + List> rows = new ArrayList<>(); + for (String tier : new String[] {"persistent", "transient"}) { + Object section = body.get(tier); + if (section instanceof Map) { + for (Map.Entry e : ((Map) section).entrySet()) { + Map row = new HashMap<>(); + row.put("setting", e.getKey()); + row.put("value", e.getValue() == null ? null : e.getValue().toString()); + row.put("tier", tier); + rows.add(row); + } + } + } + return rows; + } + + /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ + @SuppressWarnings("unchecked") + private Map getJsonMap(String path, Map params) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + if (params != null) { + params.forEach(request::addParameter); + } + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + return parser.map(); + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Long asLong(Object value) { + if (value == null) { + return null; + } + try { + return (long) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + Map body = + getJsonMap( + "/_resolve/index/*", + expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); + List> rows = new ArrayList<>(); + addResolved(body.get("indices"), "index", rows); + addResolved(body.get("aliases"), "alias", rows); + addResolved(body.get("data_streams"), "data_stream", rows); + return rows; + } + + @SuppressWarnings("unchecked") + private void addResolved(Object section, String type, List> rows) { + if (section instanceof List) { + for (Object o : (List) section) { + if (o instanceof Map) { + Map row = new HashMap<>(); + row.put("name", ((Map) o).get("name")); + row.put("type", type); + rows.add(row); + } + } + } + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new HashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java new file mode 100644 index 00000000000..282ba52b1ee --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.rest.CalciteEnumerableRestScan; +import org.opensearch.sql.opensearch.storage.rest.CalciteLogicalRestScan; + +/** Rule to convert a {@link CalciteLogicalRestScan} to a {@link CalciteEnumerableRestScan}. */ +public class EnumerableRestScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalRestScan.class, + s -> s.getRestTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableRestScanRule") + .withRuleFactory(EnumerableRestScanRule::new); + + protected EnumerableRestScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalRestScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalRestScan scan = (CalciteLogicalRestScan) rel; + return new CalciteEnumerableRestScan( + scan.getCluster(), scan.getHints(), scan.getTable(), scan.getRestTable(), scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..22a508b88cd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -14,6 +14,7 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule REST_SCAN_RULE = EnumerableRestScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -27,6 +28,7 @@ public class OpenSearchIndexRules { ImmutableList.of( INDEX_SCAN_RULE, SYSTEM_INDEX_SCAN_RULE, + REST_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..0ec3c33c664 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,6 +17,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.storage.rest.RestSourceTable; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; @@ -35,7 +38,9 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { + if (isRestSource(name)) { + return new RestSourceTable(client, settings, decodeRestSpec(name)); + } else if (isSystemIndex(name)) { return new OpenSearchSystemIndex(client, settings, name); } else { return new OpenSearchIndex(client, settings, name); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java new file mode 100644 index 00000000000..fe07d9c584e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static java.util.Objects.requireNonNull; + +import java.util.List; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; + +/** An abstract relational operator representing a scan of a {@link RestSourceTable}. */ +@Getter +public abstract class AbstractCalciteRestScan extends TableScan { + protected final RestSourceTable restTable; + protected final RelDataType schema; + + protected AbstractCalciteRestScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super(cluster, traitSet, hints, table); + this.restTable = requireNonNull(restTable, "rest source table"); + this.schema = schema; + } + + @Override + public RelDataType deriveRowType() { + return this.schema; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java new file mode 100644 index 00000000000..a55d29e785a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.PhysType; +import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.tree.Blocks; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.DeriveMode; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.util.Pair; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.plan.Scannable; + +/** The physical relational operator representing a scan of a {@link RestSourceTable}. */ +public class CalciteEnumerableRestScan extends AbstractCalciteRestScan + implements EnumerableRel, Scannable { + public CalciteEnumerableRestScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super( + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + restTable, + schema); + } + + @Override + public @Nullable Pair> passThroughTraits(RelTraitSet required) { + return EnumerableRel.super.passThroughTraits(required); + } + + @Override + public @Nullable Pair> deriveTraits( + RelTraitSet childTraits, int childId) { + return EnumerableRel.super.deriveTraits(childTraits, childId); + } + + @Override + public DeriveMode getDeriveMode() { + return EnumerableRel.super.getDeriveMode(); + } + + @Override + public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + PhysType physType = + PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); + + Expression scanOperator = implementor.stash(this, CalciteEnumerableRestScan.class); + return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); + } + + @Override + public Enumerable<@Nullable Object> scan() { + return new AbstractEnumerable<>() { + @Override + public Enumerator enumerator() { + return new RestEnumerator( + getFieldPath(), + restTable.createRestRequest(), + restTable.createOpenSearchResourceMonitor()); + } + }; + } + + private List getFieldPath() { + return getRowType().getFieldNames().stream().toList(); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java new file mode 100644 index 00000000000..4cb33b48646 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.opensearch.planner.rules.EnumerableRestScanRule; + +/** The logical relational operator representing a scan of a {@link RestSourceTable}. */ +public class CalciteLogicalRestScan extends AbstractCalciteRestScan { + + public CalciteLogicalRestScan( + RelOptCluster cluster, RelOptTable table, RestSourceTable restTable) { + this( + cluster, + cluster.traitSetOf(Convention.NONE), + ImmutableList.of(), + table, + restTable, + table.getRowType()); + } + + protected CalciteLogicalRestScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super(cluster, traitSet, hints, table, restTable, schema); + } + + @Override + public void register(RelOptPlanner planner) { + super.register(planner); + planner.addRule(EnumerableRestScanRule.DEFAULT_CONFIG.toRule()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..9b79e8e57d4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,407 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; +import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps to its transport + * action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so the Calcite plan + * can fix its row type at plan time), the query args it accepts, and a secret-field filter. + * + *

This is the single place the read-only allow-list and secret filtering are enforced. + * Endpoints outside the registry, including every mutating endpoint, are rejected by {@link + * #resolve} with a clear exception. Adding an endpoint is a reviewed change here, never arbitrary + * pass-through. + */ +public final class RestEndpointRegistry { + + private RestEndpointRegistry() {} + + /** Produces the raw rows for an endpoint via a read-only client call. */ + @FunctionalInterface + public interface RowFetcher { + List> fetch(OpenSearchClient client, RestSpec spec); + } + + /** A single allow-listed endpoint description. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final Set allowedArgs; + private final Set secretFields; + private final RowFetcher fetcher; + + Endpoint( + String path, + LinkedHashMap schema, + Set allowedArgs, + Set secretFields, + RowFetcher fetcher) { + this.path = path; + this.schema = schema; + this.allowedArgs = allowedArgs; + this.secretFields = secretFields; + this.fetcher = fetcher; + } + + /** Dispatch the read-only call and shape the response into fixed-schema rows. */ + public List toRows(OpenSearchClient client, RestSpec spec) { + List out = new ArrayList<>(); + for (Map raw : fetcher.fetch(client, spec)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + for (Map.Entry col : schema.entrySet()) { + if (secretFields.contains(col.getKey())) { + // Never surface a secret-bearing field, even if the action returns it. + continue; + } + tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); + } + out.add(new ExprTupleValue(tuple)); + } + return out; + } + } + + private static final Map REGISTRY = buildRegistry(); + + private static Map buildRegistry() { + Map m = new LinkedHashMap<>(); + + // /_cluster/health — single-row cluster health snapshot (read-only monitor action). + LinkedHashMap healthSchema = new LinkedHashMap<>(); + healthSchema.put("cluster_name", STRING); + healthSchema.put("status", STRING); + healthSchema.put("number_of_nodes", INTEGER); + healthSchema.put("number_of_data_nodes", INTEGER); + healthSchema.put("active_primary_shards", INTEGER); + healthSchema.put("active_shards", INTEGER); + healthSchema.put("relocating_shards", INTEGER); + healthSchema.put("initializing_shards", INTEGER); + healthSchema.put("unassigned_shards", INTEGER); + healthSchema.put("timed_out", BOOLEAN); + m.put( + "/_cluster/health", + new Endpoint( + "/_cluster/health", + healthSchema, + Set.of("local"), + Set.of(), + (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); + + // /_cat/indices — one row per index (read-only monitor action). + LinkedHashMap catSchema = new LinkedHashMap<>(); + catSchema.put("index", STRING); + catSchema.put("health", STRING); + catSchema.put("pri", INTEGER); + catSchema.put("rep", INTEGER); + catSchema.put("active_shards", INTEGER); + m.put( + "/_cat/indices", + new Endpoint( + "/_cat/indices", + catSchema, + Set.of("health"), + Set.of(), + (client, spec) -> client.catIndices(spec.getArgs()))); + + // /_cat/nodes — one row per node with resource state (read-only monitor action). + LinkedHashMap nodesSchema = new LinkedHashMap<>(); + nodesSchema.put("name", STRING); + nodesSchema.put("ip", STRING); + nodesSchema.put("node_role", STRING); + nodesSchema.put("heap_percent", INTEGER); + nodesSchema.put("ram_percent", INTEGER); + nodesSchema.put("cpu", INTEGER); + m.put( + "/_cat/nodes", + new Endpoint( + "/_cat/nodes", + nodesSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catNodes(spec.getArgs()))); + + // /_cat/cluster_manager — single row identifying the elected cluster manager node. + LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); + clusterManagerSchema.put("id", STRING); + clusterManagerSchema.put("host", STRING); + clusterManagerSchema.put("ip", STRING); + clusterManagerSchema.put("node", STRING); + m.put( + "/_cat/cluster_manager", + new Endpoint( + "/_cat/cluster_manager", + clusterManagerSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catClusterManager(spec.getArgs()))); + + // /_cat/plugins — one row per installed plugin per node (read-only monitor action). + LinkedHashMap pluginsSchema = new LinkedHashMap<>(); + pluginsSchema.put("name", STRING); + pluginsSchema.put("component", STRING); + pluginsSchema.put("version", STRING); + m.put( + "/_cat/plugins", + new Endpoint( + "/_cat/plugins", + pluginsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catPlugins(spec.getArgs()))); + + // /_cat/shards — one row per shard (read-only monitor action). + LinkedHashMap shardsSchema = new LinkedHashMap<>(); + shardsSchema.put("index", STRING); + shardsSchema.put("shard", INTEGER); + shardsSchema.put("prirep", STRING); + shardsSchema.put("state", STRING); + shardsSchema.put("node", STRING); + m.put( + "/_cat/shards", + new Endpoint( + "/_cat/shards", + shardsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catShards(spec.getArgs()))); + + // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). + LinkedHashMap stateSchema = new LinkedHashMap<>(); + stateSchema.put("cluster_name", STRING); + stateSchema.put("state_uuid", STRING); + stateSchema.put("version", LONG); + stateSchema.put("cluster_manager_node", STRING); + m.put( + "/_cluster/state", + new Endpoint( + "/_cluster/state", + stateSchema, + Set.of(), + Set.of(), + (client, spec) -> List.of(client.clusterState(spec.getArgs())))); + + // /_cluster/settings — one row per configured setting (persistent/transient tier). + LinkedHashMap settingsSchema = new LinkedHashMap<>(); + settingsSchema.put("setting", STRING); + settingsSchema.put("value", STRING); + settingsSchema.put("tier", STRING); + m.put( + "/_cluster/settings", + new Endpoint( + "/_cluster/settings", + settingsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.clusterSettings(spec.getArgs()))); + + // /_resolve/index — one row per resolved index/alias/data_stream name. + LinkedHashMap resolveSchema = new LinkedHashMap<>(); + resolveSchema.put("name", STRING); + resolveSchema.put("type", STRING); + m.put( + "/_resolve/index", + new Endpoint( + "/_resolve/index", + resolveSchema, + Set.of("expand_wildcards"), + Set.of(), + (client, spec) -> client.resolveIndex(spec.getArgs()))); + + return m; + } + + /** + * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public static Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Supported read-only endpoints: " + + REGISTRY.keySet()); + } + Endpoint endpoint = REGISTRY.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + REGISTRY.keySet()); + } + return endpoint; + } + + /** Validate that every supplied query arg is accepted by the endpoint. */ + public static void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + for (String arg : spec.getArgs().keySet()) { + if (!endpoint.getAllowedArgs().contains(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + endpoint.getAllowedArgs()); + } + validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } + + // Allowed value domains for the get-args that are applied server-side. Keys are validated against + // the per-endpoint allow-list above; values are validated here so a user-supplied value is never + // passed unchecked into an admin transport request. + private static final Map> ARG_VALUE_DOMAINS = + Map.of( + "local", Set.of("true", "false"), + "health", Set.of("green", "yellow", "red")); + + private static final Set EXPAND_WILDCARDS_VALUES = + Set.of("open", "closed", "hidden", "none", "all"); + + /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ + private static void validateArgValue(String endpoint, String arg, String value) { + Set domain = ARG_VALUE_DOMAINS.get(arg); + if (domain != null) { + if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + } else if ("expand_wildcards".equals(arg)) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { + if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + } + } + } + + private static ExprValue coerce(String column, ExprType type, Object value) { + if (value == null) { + return ExprNullValue.of(); + } + try { + if (type == INTEGER) { + return integerValue(toNumber(value).intValue()); + } + if (type == LONG) { + return longValue(toNumber(value).longValue()); + } + if (type == DOUBLE) { + return doubleValue(toNumber(value).doubleValue()); + } + if (type == BOOLEAN) { + return booleanValue(toBoolean(value)); + } + } catch (RuntimeException e) { + // Surface a clear client error (HTTP 400) instead of a raw ClassCastException / + // NumberFormatException (HTTP 500) when an endpoint returns an unexpected value shape. + throw new IllegalArgumentException( + "rest endpoint value for column [" + + column + + "] could not be coerced to " + + type + + ": [" + + value + + "]"); + } + return stringValue(String.valueOf(value)); + } + + /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ + private static Number toNumber(Object value) { + if (value instanceof Number n) { + return n; + } + String s = String.valueOf(value).trim(); + if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { + return Double.parseDouble(s); + } + return Long.parseLong(s); + } + + /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ + private static boolean toBoolean(Object value) { + if (value instanceof Boolean b) { + return b; + } + String s = String.valueOf(value).trim(); + if (s.equalsIgnoreCase("true")) { + return true; + } + if (s.equalsIgnoreCase("false")) { + return false; + } + throw new IllegalArgumentException("not a boolean: " + value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java new file mode 100644 index 00000000000..ad678acb4c4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Iterator; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.calcite.linq4j.Enumerator; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.NonFallbackCalciteException; +import org.opensearch.sql.monitor.ResourceMonitor; + +/** + * A simple resource-monitored iteration over the rows produced by a {@code rest} endpoint dispatch. + * One-for-one parallel of {@code OpenSearchSystemIndexEnumerator}. + */ +public class RestEnumerator implements Enumerator { + /** How many moveNext() calls to perform a resource check once. */ + private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; + + private final List fields; + + @EqualsAndHashCode.Include @ToString.Include private final RestRequest request; + + private Iterator iterator; + + private ExprValue current; + + /** Number of rows returned. */ + private Integer queryCount; + + /** ResourceMonitor. */ + private final ResourceMonitor monitor; + + public RestEnumerator(List fields, RestRequest request, ResourceMonitor monitor) { + this.fields = fields; + this.request = request; + this.monitor = monitor; + this.queryCount = 0; + this.current = null; + if (!this.monitor.isHealthy()) { + throw new NonFallbackCalciteException("insufficient resources to run the query, quit."); + } + this.iterator = request.search().iterator(); + } + + @Override + public Object current() { + return fields.stream() + .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) + .toArray(); + } + + @Override + public boolean moveNext() { + boolean shouldCheck = (queryCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); + if (shouldCheck && !this.monitor.isHealthy()) { + throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); + } + if (iterator.hasNext()) { + current = iterator.next(); + queryCount++; + return true; + } else { + return false; + } + } + + @Override + public void reset() { + iterator = request.search().iterator(); + queryCount = 0; + current = null; + } + + @Override + public void close() { + iterator = null; + current = null; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..d6ee4dff1ac --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + } + + @Override + public List search() { + List rows = endpoint.toRows(client, spec); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java new file mode 100644 index 00000000000..7269dce4955 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on + * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the + * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only allow-list and + * secret filtering), exposes the endpoint's fixed schema, and produces a {@link + * CalciteLogicalRestScan} on the Calcite path. + */ +@Getter +public class RestSourceTable extends AbstractOpenSearchTable { + + private final OpenSearchClient client; + private final Settings settings; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + + public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec) { + this.client = client; + this.settings = settings; + this.spec = spec; + // Allow-list and secret filtering enforced here: unknown/mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("rest endpoint is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalRestScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } + + public RestRequest createRestRequest() { + return new RestRequest(client, endpoint, spec); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..4284aa6d25d --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,213 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestEndpointRegistryTest { + + @Mock private OpenSearchClient client; + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint is simply absent from the registry and is refused here. + assertThrows( + IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); + assertThrows( + IllegalArgumentException.class, + () -> RestEndpointRegistry.resolve("/services/server/info")); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + RestEndpointRegistry.validate(spec); // no throw + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsDroppedFlatSettingsArg() { + // flat_settings was dropped (redundant: settings are already flattened to dotted keys). + RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsValidArgValues() { + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException health = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); + assertTrue(health.getMessage().contains("unsupported value")); + + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); + RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void coerceParsesNumericStringValues() { + // The cat JSON API returns numeric columns as strings; coerce must parse them. + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "3"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void coerceThrowsClearErrorOnUncoercibleValue() { + // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), + // not a raw ClassCastException / NumberFormatException (HTTP 500). + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "not-a-number"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + assertTrue(ex.getMessage().contains("number_of_nodes")); + assertTrue(ex.getMessage().contains("not-a-number")); + } + + @Test + void clusterHealthRowsAreShapedToFixedSchema() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "test-cluster"); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + // a declared column the action did not return becomes null, never absent. + assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + } + + @Test + void catIndicesRowsAreShapedToFixedSchema() { + Map idx = new LinkedHashMap<>(); + idx.put("index", "books"); + idx.put("health", "yellow"); + idx.put("pri", 1); + idx.put("rep", 1); + when(client.catIndices(any())).thenReturn(List.of(idx)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); + List rows = + endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); + assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java new file mode 100644 index 00000000000..a50bf27f042 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import com.google.common.collect.ImmutableMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestSourceTableTest { + + @Mock private OpenSearchClient client; + + @Mock private Settings settings; + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + Map fieldTypes = table.getFieldTypes(); + assertThat(fieldTypes, hasEntry("status", STRING)); + assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + } + + @Test + void existsIsTrue() { + Table table = new RestSourceTable(client, settings, healthSpec()); + assertTrue(table.exists()); + } + + @Test + void createIsUnsupported() { + Table table = new RestSourceTable(client, settings, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> table.create(ImmutableMap.of())); + } + + @Test + void implementIsUnsupportedOnV2() { + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> table.implement(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, + settings, + new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cat/indices", Map.of(), -1, null))); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + } + + @Test + void restRequestShapesResponseRows() { + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + List rows = table.createRestRequest().search(); + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void countTruncatesRows() { + Map idx1 = new LinkedHashMap<>(); + idx1.put("index", "a"); + Map idx2 = new LinkedHashMap<>(); + idx2.put("index", "b"); + when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); + + RestSourceTable table = + new RestSourceTable(client, settings, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = table.createRestRequest().search(); + assertEquals(1, rows.size()); + assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4bc69a8f295..197886342e7 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 98f85b08282..0d4469893c2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -103,6 +104,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -206,6 +208,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3741137f5a9..546450b8191 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -109,6 +109,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -140,6 +141,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -251,6 +253,37 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
+ * Leading command that reads an allow-listed, read-only in-cluster management endpoint + * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved + * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves + * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE + * resolves to a system index. Allow-list/authorization enforcement happens at source-table + * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 4b75d444467..fbd447c4e0d 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -96,6 +96,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -122,6 +123,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -165,6 +167,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..790c071f6ae --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} + * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch + * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is exercised in {@code RestSourceTableTest} (logical scan + fixed row type, + * unit) and {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test + * pins the Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code + * rest} into a {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec + * that rides {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = + (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(2, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d57ca8a69bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,4 +1939,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6756cdc198a..21c333166f1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,6 +48,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); From 7580a51eca48d4c7bc71e95e8f5a00504fb4b46e Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 09:46:31 -0700 Subject: [PATCH 02/23] Align rest '/_cluster/settings' redaction with native endpoint; CI fixes; comment cleanup - /_cluster/settings: run the persistent and transient tiers through the node SettingsFilter (published via RestSettingsFilterHolder from SQLPlugin#getRestHandlers) so Property.Filtered and plugin-registered pattern settings are redacted exactly as the native GET /_cluster/settings endpoint. Remove the dead secretFields column-filter, which was the wrong shape for the (setting, value, tier) rows. - Parser: add TIMEOUT to searchableKeyWord so a bare 'timeout' term still matches searchLiteral. - coerce(): narrow the catch to IllegalArgumentException | ClassCastException; add empty-string guards in toNumber/toBoolean. - spotlessApply formatting; drop outdated and redundant comments. Tests: RestEndpointRegistryTest, RestSourceTableTest, OpenSearchNodeClientClusterSettingsFilterTest green. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalcitePPLRestIT.java | 17 ++- .../client/OpenSearchNodeClient.java | 21 +++- .../client/OpenSearchRestClient.java | 3 +- .../storage/rest/RestEndpointRegistry.java | 47 ++++---- .../rest/RestSettingsFilterHolder.java | 40 +++++++ .../storage/rest/RestSourceTable.java | 9 +- ...chNodeClientClusterSettingsFilterTest.java | 102 ++++++++++++++++++ .../rest/RestEndpointRegistryTest.java | 3 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + .../sql/ppl/calcite/CalcitePPLRestTest.java | 11 +- 11 files changed, 205 insertions(+), 54 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java index c2ea6069996..70ee4ef0de6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -19,7 +19,8 @@ /** * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also verifies that a non-allow-listed / mutating endpoint is refused. + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also + * verifies that a non-allow-listed / mutating endpoint is refused. */ public class CalcitePPLRestIT extends PPLIntegTestCase { @@ -31,8 +32,7 @@ public void init() throws Exception { @Test public void testRestClusterHealthSchema() throws IOException { - JSONObject result = - executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); } @@ -70,13 +70,12 @@ public void testRestRejectsTimeoutArg() throws IOException { /** * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) - * with the given substring in the response body. The negative-case check for allow-list and secret-filter enforcement. + * with the given substring in the response body. Covers allow-list and bad-argument rejection. */ private void assertRestBadRequest(String query, String expectedSubstring) { ResponseException e = org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); - org.junit.Assert.assertEquals( - 400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); org.junit.Assert.assertTrue( "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), e.getMessage().contains(expectedSubstring)); @@ -153,8 +152,7 @@ public void testRestClusterStateSingleRow() throws IOException { @Test public void testRestClusterSettingsSchema() throws IOException { // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = - executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); verifySchema( result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); } @@ -187,8 +185,7 @@ public void testRestClusterHealthLocalArg() throws IOException { @Test public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = - executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); verifyDataRows(result, rows(0)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 9ce9ff79ea8..bc5ac2128ea 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -459,8 +459,22 @@ public List> clusterSettings(Map params) { .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) .actionGet(); List> rows = new java.util.ArrayList<>(); - collectSettings(response.getState().metadata().persistentSettings(), "persistent", rows); - collectSettings(response.getState().metadata().transientSettings(), "transient", rows); + org.opensearch.common.settings.Settings persistent = + response.getState().metadata().persistentSettings(); + org.opensearch.common.settings.Settings transientSettings = + response.getState().metadata().transientSettings(); + // Mirror the native GET /_cluster/settings redaction: run both tiers through the node's + // SettingsFilter so Property.Filtered (and plugin-registered pattern) settings are not + // surfaced raw. The transport path carries no SettingsFilter of its own; the node instance is + // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter != null) { + persistent = filter.filter(persistent); + transientSettings = filter.filter(transientSettings); + } + collectSettings(persistent, "persistent", rows); + collectSettings(transientSettings, "transient", rows); return rows; } @@ -498,7 +512,8 @@ public List> resolveIndex(Map params) { .DEFAULT_INDICES_OPTIONS)); org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = client - .execute(org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .execute( + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) .actionGet(); List> rows = new java.util.ArrayList<>(); for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index c9290408229..5afe3a927f1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -285,8 +285,7 @@ public Map clusterHealth(Map params) { if (params != null && Boolean.parseBoolean(params.get("local"))) { request.local(true); } - ClusterHealthResponse response = - client.cluster().health(request, RequestOptions.DEFAULT); + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); return flattenHealth(response); } catch (IOException e) { throw new IllegalStateException("Failed to get cluster health", e); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index 9b79e8e57d4..feb6a6fafe3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -30,14 +30,13 @@ import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps to its transport - * action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so the Calcite plan - * can fix its row type at plan time), the query args it accepts, and a secret-field filter. + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps + * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so + * the Calcite plan can fix its row type at plan time), and the query args it accepts. * - *

This is the single place the read-only allow-list and secret filtering are enforced. - * Endpoints outside the registry, including every mutating endpoint, are rejected by {@link - * #resolve} with a clear exception. Adding an endpoint is a reviewed change here, never arbitrary - * pass-through. + *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, + * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. + * Adding an endpoint is a reviewed change here, never arbitrary pass-through. */ public final class RestEndpointRegistry { @@ -55,19 +54,16 @@ public static final class Endpoint { private final String path; private final LinkedHashMap schema; private final Set allowedArgs; - private final Set secretFields; private final RowFetcher fetcher; Endpoint( String path, LinkedHashMap schema, Set allowedArgs, - Set secretFields, RowFetcher fetcher) { this.path = path; this.schema = schema; this.allowedArgs = allowedArgs; - this.secretFields = secretFields; this.fetcher = fetcher; } @@ -77,10 +73,6 @@ public List toRows(OpenSearchClient client, RestSpec spec) { for (Map raw : fetcher.fetch(client, spec)) { LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { - if (secretFields.contains(col.getKey())) { - // Never surface a secret-bearing field, even if the action returns it. - continue; - } tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); } out.add(new ExprTupleValue(tuple)); @@ -112,7 +104,6 @@ private static Map buildRegistry() { "/_cluster/health", healthSchema, Set.of("local"), - Set.of(), (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); // /_cat/indices — one row per index (read-only monitor action). @@ -128,7 +119,6 @@ private static Map buildRegistry() { "/_cat/indices", catSchema, Set.of("health"), - Set.of(), (client, spec) -> client.catIndices(spec.getArgs()))); // /_cat/nodes — one row per node with resource state (read-only monitor action). @@ -145,7 +135,6 @@ private static Map buildRegistry() { "/_cat/nodes", nodesSchema, Set.of(), - Set.of(), (client, spec) -> client.catNodes(spec.getArgs()))); // /_cat/cluster_manager — single row identifying the elected cluster manager node. @@ -160,7 +149,6 @@ private static Map buildRegistry() { "/_cat/cluster_manager", clusterManagerSchema, Set.of(), - Set.of(), (client, spec) -> client.catClusterManager(spec.getArgs()))); // /_cat/plugins — one row per installed plugin per node (read-only monitor action). @@ -174,7 +162,6 @@ private static Map buildRegistry() { "/_cat/plugins", pluginsSchema, Set.of(), - Set.of(), (client, spec) -> client.catPlugins(spec.getArgs()))); // /_cat/shards — one row per shard (read-only monitor action). @@ -190,7 +177,6 @@ private static Map buildRegistry() { "/_cat/shards", shardsSchema, Set.of(), - Set.of(), (client, spec) -> client.catShards(spec.getArgs()))); // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). @@ -205,7 +191,6 @@ private static Map buildRegistry() { "/_cluster/state", stateSchema, Set.of(), - Set.of(), (client, spec) -> List.of(client.clusterState(spec.getArgs())))); // /_cluster/settings — one row per configured setting (persistent/transient tier). @@ -219,7 +204,6 @@ private static Map buildRegistry() { "/_cluster/settings", settingsSchema, Set.of(), - Set.of(), (client, spec) -> client.clusterSettings(spec.getArgs()))); // /_resolve/index — one row per resolved index/alias/data_stream name. @@ -232,7 +216,6 @@ private static Map buildRegistry() { "/_resolve/index", resolveSchema, Set.of("expand_wildcards"), - Set.of(), (client, spec) -> client.resolveIndex(spec.getArgs()))); return m; @@ -276,9 +259,7 @@ public static void validate(RestSpec spec) { // cluster-manager vs client socket timeouts differ per action). Reject it with a clear // client error rather than silently ignoring it. throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] does not support the timeout argument yet"); + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); } if (spec.getArgs() != null) { for (String arg : spec.getArgs().keySet()) { @@ -363,9 +344,11 @@ private static ExprValue coerce(String column, ExprType type, Object value) { if (type == BOOLEAN) { return booleanValue(toBoolean(value)); } - } catch (RuntimeException e) { - // Surface a clear client error (HTTP 400) instead of a raw ClassCastException / - // NumberFormatException (HTTP 500) when an endpoint returns an unexpected value shape. + } catch (IllegalArgumentException | ClassCastException e) { + // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint + // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, + // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely + // unexpected faults (NPE, etc.) are left to propagate. throw new IllegalArgumentException( "rest endpoint value for column [" + column @@ -384,6 +367,9 @@ private static Number toNumber(Object value) { return n; } String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { return Double.parseDouble(s); } @@ -396,6 +382,9 @@ private static boolean toBoolean(Object value) { return b; } String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new IllegalArgumentException("empty string is not a boolean"); + } if (s.equalsIgnoreCase("true")) { return true; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java new file mode 100644 index 00000000000..d425c361e6a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.common.settings.SettingsFilter; + +/** + * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' + * /_cluster/settings'} fetcher. + * + *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with + * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the + * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads + * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the + * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction + * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. + * + *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in + * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link + * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code + * getRestHandlers} fires lets the fetcher read the same instance without going back through the + * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. + */ +public final class RestSettingsFilterHolder { + + private static volatile SettingsFilter settingsFilter; + + private RestSettingsFilterHolder() {} + + public static void set(SettingsFilter instance) { + settingsFilter = instance; + } + + public static SettingsFilter get() { + return settingsFilter; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java index 7269dce4955..a900a1d319e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java @@ -23,9 +23,9 @@ /** * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the - * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only allow-list and - * secret filtering), exposes the endpoint's fixed schema, and produces a {@link - * CalciteLogicalRestScan} on the Calcite path. + * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only + * allow-list), exposes the endpoint's fixed schema, and produces a {@link CalciteLogicalRestScan} + * on the Calcite path. */ @Getter public class RestSourceTable extends AbstractOpenSearchTable { @@ -39,7 +39,8 @@ public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec this.client = client; this.settings = settings; this.spec = spec; - // Allow-list and secret filtering enforced here: unknown/mutating endpoints and disallowed args are rejected. + // Allow-list enforced here: unknown/mutating endpoints and disallowed args + // are rejected. this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); RestEndpointRegistry.validate(spec); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java new file mode 100644 index 00000000000..47ea968cb8f --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.client; + +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using + * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. + */ +class OpenSearchNodeClientClusterSettingsFilterTest { + + @AfterEach + void clearHolder() { + RestSettingsFilterHolder.set(null); + } + + private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) + .thenReturn(stateResp); + when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); + when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); + return new OpenSearchNodeClient(nodeClient); + } + + @Test + void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("plugins.secret.token", "supersecret") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Publish a filter that redacts the secret key, exactly as the native endpoint would. + RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); + + List> rows = client.clusterSettings(Map.of()); + Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); + assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); + } + + @Test + void clusterSettingsRedactsByGlobPattern() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("s3.client.default.secret_key", "AKIAEXAMPLE") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); + + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable")); + assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); + } + + @Test + void clusterSettingsReturnsRawWhenNoFilterPublished() { + Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // No filter published: confirms the SettingsFilter is the redaction mechanism (not some other + // code path). At runtime getRestHandlers always publishes the filter before any query. + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("plugins.secret.token")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index 4284aa6d25d..3254ee12f09 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -107,7 +107,8 @@ void validateRejectsBadArgValue() { IllegalArgumentException.class, () -> RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + new RestSpec( + "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); } @Test diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index ab8b923e3ae..48ad5c0b0a8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -197,6 +197,10 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); + // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can + // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); + return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 0d4469893c2..8dca6a54ba0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1785,4 +1785,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT ; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index 790c071f6ae..f26a2e617e4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -24,11 +24,12 @@ *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch - * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is exercised in {@code RestSourceTableTest} (logical scan + fixed row type, - * unit) and {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test - * pins the Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code - * rest} into a {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec - * that rides {@code visitRelation} exactly like {@code DESCRIBE}. + * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is + * exercised in {@code RestSourceTableTest} (logical scan + fixed row type, unit) and {@code + * CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. */ public class CalcitePPLRestTest { From ba90548cb99bff9355cc7c760e7d4ec7df6bf339 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 10:18:53 -0700 Subject: [PATCH 03/23] Address rest command bot-review findings; register rest doctest - clusterSettings: fail closed (throw IllegalStateException) when the node SettingsFilter is unavailable, instead of returning unredacted settings - collectSettings: handle list-type settings via getAsList fallback - decodeRestSpec: reject a blank/missing endpoint with a clear error - docs: correct rest.md allow-list table (9 endpoints + accepted args), quote endpoint literals, fix timeout/get-arg descriptions, add security note - register docs/user/ppl/cmd/rest.md in docs/category.json (deterministic single-node examples: number_of_nodes=1, cluster_manager count=1) Signed-off-by: Louis Chu --- .../sql/utils/SystemIndexUtils.java | 3 + docs/category.json | 1 + docs/user/ppl/cmd/rest.md | 58 +++++++++++++------ .../client/OpenSearchNodeClient.java | 21 +++++-- ...chNodeClientClusterSettingsFilterTest.java | 15 ++--- 5 files changed, 66 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 200a9f19681..25ea68005d8 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -104,6 +104,9 @@ public static RestSpec decodeRestSpec(String indexName) { args.put(k.substring("arg.".length()), v); } } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } return new RestSpec(endpoint, args, count, timeout); } diff --git a/docs/category.json b/docs/category.json index ac1dda966d2..0defdaaa6d3 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index 11f35e2ee86..b97f247445f 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -2,7 +2,7 @@ The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. ## Syntax @@ -20,43 +20,63 @@ The `rest` command supports the following parameters. | --- | --- | --- | | `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | | `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Request timeout passed to the transport action, for example `30s`. | -| `=` | Optional | Endpoint query arguments, validated per endpoint (for example `level=indices` for `/_cluster/health`). | +| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | ## Allow-list `rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. -| Endpoint | Output columns | -| --- | --- | -| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | +| Endpoint | Output columns | Accepted args | +| --- | --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | +| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | +| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | +| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | +| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | +| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | +| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | +| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | -## Example 1: Reading cluster health +## Example 1: Counting the nodes in the cluster -The following query reads cluster health and projects two columns: +The following query reads cluster health and projects a column that is deterministic on a single-node cluster: ```ppl -| rest /_cluster/health | fields status, number_of_nodes +| rest '/_cluster/health' | fields number_of_nodes ``` The query returns the following results: ```text fetched rows / total rows = 1/1 -+--------+-----------------+ -| status | number_of_nodes | -|--------+-----------------| -| green | 1 | -+--------+-----------------+ ++-----------------+ +| number_of_nodes | +|-----------------| +| 1 | ++-----------------+ ``` -## Example 2: Listing indexes from cat indices +`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. + +## Example 2: Composing downstream commands over a cat endpoint -The following query lists indexes and filters and sorts them on the Calcite plan: +The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: ```ppl -| rest /_cat/indices | where health = "green" | sort index | fields index, health, pri +| rest '/_cat/cluster_manager' | stats count() as managers +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| managers | +|----------| +| 1 | ++----------+ ``` -The downstream `where`, `sort`, and `fields` compose over the `rest` row source exactly like any index scan. +For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index bc5ac2128ea..8efeda5f3cf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -469,10 +469,17 @@ public List> clusterSettings(Map params) { // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. org.opensearch.common.settings.SettingsFilter filter = org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter != null) { - persistent = filter.filter(persistent); - transientSettings = filter.filter(transientSettings); + if (filter == null) { + // Fail closed: without the redaction filter (which strips Property.Filtered and + // plugin-registered secret-bearing keys) the command must refuse rather than surface raw + // settings. In-cluster the filter is published at startup (SQLPlugin#getRestHandlers) before + // any query runs, so this guards only against an unexpected uninitialized state. + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); } + persistent = filter.filter(persistent); + transientSettings = filter.filter(transientSettings); collectSettings(persistent, "persistent", rows); collectSettings(transientSettings, "transient", rows); return rows; @@ -488,7 +495,13 @@ private void collectSettings( for (String key : settings.keySet()) { Map row = new java.util.LinkedHashMap<>(); row.put("setting", key); - row.put("value", settings.get(key)); + String value = settings.get(key); + if (value == null) { + // List-valued settings return null from get(); fall back to the joined list form. + java.util.List list = settings.getAsList(key); + value = list.isEmpty() ? null : String.join(",", list); + } + row.put("value", value); row.put("tier", tier); rows.add(row); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java index 47ea968cb8f..33be6cc8482 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -7,6 +7,7 @@ import static java.util.stream.Collectors.toSet; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; import static org.mockito.ArgumentMatchers.any; @@ -86,17 +87,13 @@ void clusterSettingsRedactsByGlobPattern() { } @Test - void clusterSettingsReturnsRawWhenNoFilterPublished() { + void clusterSettingsFailsClosedWhenNoFilterPublished() { Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - // No filter published: confirms the SettingsFilter is the redaction mechanism (not some other - // code path). At runtime getRestHandlers always publishes the filter before any query. - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("plugins.secret.token")); + // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw + // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter + // before any query, so this path is unreachable in cluster. + assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); } } From cc58b4ca93d9ecf723b7aa66dea3550389d740b6 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 10:50:41 -0700 Subject: [PATCH 04/23] Harden decodeRestSpec: reject non-rest-source tokens with a clear error decodeRestSpec is only ever called behind an isRestSource gate today, but as a public decoder it must not assume its precondition. Without the guard a malformed token would surface an opaque StringIndexOutOfBoundsException from substring; now it throws a clear IllegalArgumentException instead. Addresses the PR #5599 Code Suggestions finding (importance 8). Signed-off-by: Louis Chu --- .../java/org/opensearch/sql/utils/SystemIndexUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 25ea68005d8..4df2f34c78c 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -76,6 +76,12 @@ public static String restTable(RestSpec spec) { /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } String body = indexName.substring( REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); From eed96213b103ad4c67f8207c238e1977dab8b322 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 19:21:06 -0700 Subject: [PATCH 05/23] Fix rest explain IT (JSON payload) and harden cluster-settings/state fetch - CalciteExplainIT.explainRestCommand: single-quote the endpoint literal; the explain harness inlines the query into a JSON body without escaping, so a double-quoted literal produced an invalid payload and a 400 (integration CI failure). - OpenSearchNodeClient.clusterSettings: resolve the SettingsFilter and fail closed BEFORE fetching cluster state, so settings are never read into memory when the redaction filter is unavailable. - OpenSearchRestClient.clusterState: narrow filter_path to nodes.*.name so node IPs/attributes are not over-fetched; manager-name resolution is preserved. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalciteExplainIT.java | 2 +- .../sql/ppl/NewAddedCommandsIT.java | 2 +- .../opensearch/sql/sql/VectorSearchIT.java | 1 + .../client/OpenSearchNodeClient.java | 31 +++++++------------ .../client/OpenSearchRestClient.java | 5 ++- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 784e324bfbd..f8623c70d6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -67,7 +67,7 @@ public void init() throws Exception { // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. @Test public void explainRestCommand() throws IOException { - String result = explainQueryToString("| rest \"/_cluster/health\" | fields status"); + String result = explainQueryToString("| rest '/_cluster/health' | fields status"); Assert.assertTrue( "Expected a rest scan node in the explain output, got: " + result, result.contains("RestScan") || result.contains("rest")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 8d5912ab199..2ccda31eea7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -36,7 +36,7 @@ public void init() throws Exception { public void testRest() throws IOException { JSONObject result; try { - result = executeQuery("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 8ae3167b40b..e08735ebda7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -25,6 +25,7 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 8efeda5f3cf..080a8627894 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -452,6 +452,16 @@ public Map clusterState(Map params) { @Override public List> clusterSettings(Map params) { + // The transport path has no SettingsFilter of its own; it is published from + // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings + // into memory when we cannot redact them, matching native GET /_cluster/settings. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter == null) { + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); + } org.opensearch.action.admin.cluster.state.ClusterStateResponse response = client .admin() @@ -460,26 +470,9 @@ public List> clusterSettings(Map params) { .actionGet(); List> rows = new java.util.ArrayList<>(); org.opensearch.common.settings.Settings persistent = - response.getState().metadata().persistentSettings(); + filter.filter(response.getState().metadata().persistentSettings()); org.opensearch.common.settings.Settings transientSettings = - response.getState().metadata().transientSettings(); - // Mirror the native GET /_cluster/settings redaction: run both tiers through the node's - // SettingsFilter so Property.Filtered (and plugin-registered pattern) settings are not - // surfaced raw. The transport path carries no SettingsFilter of its own; the node instance is - // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - // Fail closed: without the redaction filter (which strips Property.Filtered and - // plugin-registered secret-bearing keys) the command must refuse rather than surface raw - // settings. In-cluster the filter is published at startup (SQLPlugin#getRestHandlers) before - // any query runs, so this guards only against an unexpected uninitialized state. - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - persistent = filter.filter(persistent); - transientSettings = filter.filter(transientSettings); + filter.filter(response.getState().metadata().transientSettings()); collectSettings(persistent, "persistent", rows); collectSettings(transientSettings, "transient", rows); return rows; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index 5afe3a927f1..e98c5bf95f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -423,7 +423,10 @@ public Map clusterState(Map params) { Map state = getJsonMap( "/_cluster/state/master_node,version,metadata,nodes", - Map.of("filter_path", "cluster_name,state_uuid,version,cluster_manager_node,nodes")); + // nodes.*.name resolves the manager id to a name without over-fetching node IPs. + Map.of( + "filter_path", + "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); Map row = new HashMap<>(); row.put("cluster_name", state.get("cluster_name")); row.put("state_uuid", state.get("state_uuid")); From 7af15b0513d6d31b17209f029979469ac9bcfd96 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 2 Jul 2026 07:03:19 -0700 Subject: [PATCH 06/23] Rest command: analytics-engine coexistence IT + hardened source-token decode - AnalyticsEngineCompatIT: assert | rest '/_cluster/health' behaves identically with the analytics engine enabled (rest is never routed to DataFusion). - SystemIndexUtils.fromHex: reject an odd-length hex body so a crafted source name that passes the isRestSource suffix check fails clearly rather than silently dropping the trailing half-byte. Signed-off-by: Louis Chu --- .../sql/utils/SystemIndexUtils.java | 3 ++ .../sql/plugin/AnalyticsEngineCompatIT.java | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 4df2f34c78c..7589cf522f6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -125,6 +125,9 @@ private static String toHex(String s) { } private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } byte[] bytes = new byte[h.length() / 2]; for (int i = 0; i < bytes.length; i++) { bytes[i] = diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..b5c08cc8ad0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,31 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * fixed schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(2, schema.length()); + assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); + assertEquals("int", schema.getJSONObject(1).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + } } From 7a797099d3c244b1c4a7cf3b3c4825cef3270abf Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 7 Jul 2026 19:00:46 +0800 Subject: [PATCH 07/23] [Bugfix] Keep rest command on Calcite path under cluster-composite On a cluster started with cluster.pluggable.dataformat=composite, RestUnifiedQueryAction.isAnalyticsIndex() routed every non-system-catalog PPL query to the analytics engine. The rest command's reserved in-cluster source (REST...__REST_SOURCE) has no backing index and only resolves on the Calcite path, so it was routed to DataFusion and failed with "Table 'REST...__REST_SOURCE' not found". Fix: exclude isRestSource(name) alongside isSystemCatalog(name) so a rest source falls back to the default (Calcite) pipeline and is never routed to the analytics engine. - RestUnifiedQueryActionTest: unit repro under cluster-composite. - integ-test analyticsEngineCompat testcluster set composite-default so the existing rest coexistence IT exercises this routing exclusion. Signed-off-by: Louis Chu --- integ-test/build.gradle | 2 ++ .../sql/plugin/rest/RestUnifiedQueryAction.java | 9 +++++---- .../sql/plugin/rest/RestUnifiedQueryActionTest.java | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2e565aab8f2..d04c4484cc1 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -419,6 +419,8 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" + // Composite-default cluster: PPL queries route to the analytics engine unless excluded. + setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 62f3dd0c346..943bae38dc6 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -99,13 +99,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..0cf87f0604e 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,6 +178,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); From f83b14b567828d0082bbe76ef2fa425bc10cbbc8 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 8 Jul 2026 14:28:07 +0800 Subject: [PATCH 08/23] [Refactor] Unify system-index and rest scans behind one catalog table Collapse the two near-duplicate Calcite scan hierarchies -- SHOW/DESCRIBE system tables and the rest command -- into one generic OpenSearchCatalogTable whose per-endpoint behavior is supplied by a pluggable CatalogSource. - OpenSearchSystemIndex + RestSourceTable -> one OpenSearchCatalogTable backed by SystemIndexCatalogSource / RestCatalogSource. - Two Abstract/Logical/Enumerable scans + two enumerators + two converter rules -> AbstractCalciteCatalogScan / CalciteLogicalCatalogScan / CalciteEnumerableCatalogScan (+ rest-only CalciteScannableCatalogScan) / OpenSearchCatalogEnumerator / EnumerableCatalogScanRule. - Concerns stay per-source: system tables keep the real V2 implement() path; rest is Calcite-only (implement() throws) and opts into Scannable for the collect short-circuit. No behavior change: dispatch, schemas, V2 support, and the Scannable marker are preserved; this is pure de-duplication of the Calcite scan plumbing. Verified: opensearch compileJava/compileTestJava green; ppl and integ-test test-compile green; affected unit tests pass. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalciteExplainIT.java | 4 +- .../rules/EnumerableCatalogScanRule.java | 63 +++++++++++ .../planner/rules/EnumerableRestScanRule.java | 46 -------- .../rules/EnumerableSystemIndexScanRule.java | 50 --------- .../planner/rules/OpenSearchIndexRules.java | 8 +- .../storage/OpenSearchStorageEngine.java | 10 +- .../storage/rest/AbstractCalciteRestScan.java | 41 ------- .../storage/rest/CalciteLogicalRestScan.java | 48 -------- .../storage/rest/RestCatalogSource.java | 57 ++++++++++ .../storage/rest/RestEnumerator.java | 86 -------------- .../storage/rest/RestSourceTable.java | 81 -------------- ...n.java => AbstractCalciteCatalogScan.java} | 12 +- .../CalciteEnumerableCatalogScan.java} | 24 ++-- .../CalciteEnumerableSystemIndexScan.java | 82 -------------- ...an.java => CalciteLogicalCatalogScan.java} | 20 ++-- .../system/CalciteScannableCatalogScan.java | 29 +++++ .../storage/system/CatalogSource.java | 39 +++++++ ....java => OpenSearchCatalogEnumerator.java} | 9 +- .../system/OpenSearchCatalogTable.java | 67 +++++++++++ .../storage/system/OpenSearchSystemIndex.java | 105 ------------------ .../system/SystemIndexCatalogSource.java | 70 ++++++++++++ .../storage/OpenSearchStorageEngineTest.java | 5 +- ...leTest.java => RestCatalogSourceTest.java} | 60 ++++------ ...t.java => OpenSearchCatalogTableTest.java} | 25 +++-- .../sql/ppl/calcite/CalcitePPLRestTest.java | 10 +- 25 files changed, 416 insertions(+), 635 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{AbstractCalciteSystemIndexScan.java => AbstractCalciteCatalogScan.java} (68%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/{rest/CalciteEnumerableRestScan.java => system/CalciteEnumerableCatalogScan.java} (81%) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteLogicalSystemIndexScan.java => CalciteLogicalCatalogScan.java} (59%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexEnumerator.java => OpenSearchCatalogEnumerator.java} (90%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/{RestSourceTableTest.java => RestCatalogSourceTest.java} (60%) rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexTest.java => OpenSearchCatalogTableTest.java} (76%) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index f8623c70d6b..fd8515aecfd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -64,13 +64,13 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. + // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. @Test public void explainRestCommand() throws IOException { String result = explainQueryToString("| rest '/_cluster/health' | fields status"); Assert.assertTrue( "Expected a rest scan node in the explain output, got: " + result, - result.contains("RestScan") || result.contains("rest")); + result.contains("CatalogScan")); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java deleted file mode 100644 index 282ba52b1ee..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.rest.CalciteEnumerableRestScan; -import org.opensearch.sql.opensearch.storage.rest.CalciteLogicalRestScan; - -/** Rule to convert a {@link CalciteLogicalRestScan} to a {@link CalciteEnumerableRestScan}. */ -public class EnumerableRestScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalRestScan.class, - s -> s.getRestTable() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableRestScanRule") - .withRuleFactory(EnumerableRestScanRule::new); - - protected EnumerableRestScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalRestScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalRestScan scan = (CalciteLogicalRestScan) rel; - return new CalciteEnumerableRestScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getRestTable(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 22a508b88cd..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,9 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule REST_SCAN_RULE = EnumerableRestScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -27,8 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, - REST_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 0ec3c33c664..64bc83211b1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -17,8 +17,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.rest.RestSourceTable; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; @@ -39,9 +40,10 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { if (isRestSource(name)) { - return new RestSourceTable(client, settings, decodeRestSpec(name)); + return new OpenSearchCatalogTable( + new RestCatalogSource(client, decodeRestSpec(name)), settings); } else if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java deleted file mode 100644 index fe07d9c584e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static java.util.Objects.requireNonNull; - -import java.util.List; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; - -/** An abstract relational operator representing a scan of a {@link RestSourceTable}. */ -@Getter -public abstract class AbstractCalciteRestScan extends TableScan { - protected final RestSourceTable restTable; - protected final RelDataType schema; - - protected AbstractCalciteRestScan( - RelOptCluster cluster, - RelTraitSet traitSet, - List hints, - RelOptTable table, - RestSourceTable restTable, - RelDataType schema) { - super(cluster, traitSet, hints, table); - this.restTable = requireNonNull(restTable, "rest source table"); - this.schema = schema; - } - - @Override - public RelDataType deriveRowType() { - return this.schema; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java deleted file mode 100644 index 4cb33b48646..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableRestScanRule; - -/** The logical relational operator representing a scan of a {@link RestSourceTable}. */ -public class CalciteLogicalRestScan extends AbstractCalciteRestScan { - - public CalciteLogicalRestScan( - RelOptCluster cluster, RelOptTable table, RestSourceTable restTable) { - this( - cluster, - cluster.traitSetOf(Convention.NONE), - ImmutableList.of(), - table, - restTable, - table.getRowType()); - } - - protected CalciteLogicalRestScan( - RelOptCluster cluster, - RelTraitSet traitSet, - List hints, - RelOptTable table, - RestSourceTable restTable, - RelDataType schema) { - super(cluster, traitSet, hints, table, restTable, schema); - } - - @Override - public void register(RelOptPlanner planner) { - super.register(planner); - planner.addRule(EnumerableRestScanRule.DEFAULT_CONFIG.toRule()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..807f83df4bf --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. + * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + + public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this.client = client; + this.spec = spec; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java deleted file mode 100644 index ad678acb4c4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Iterator; -import java.util.List; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.apache.calcite.linq4j.Enumerator; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.exception.NonFallbackCalciteException; -import org.opensearch.sql.monitor.ResourceMonitor; - -/** - * A simple resource-monitored iteration over the rows produced by a {@code rest} endpoint dispatch. - * One-for-one parallel of {@code OpenSearchSystemIndexEnumerator}. - */ -public class RestEnumerator implements Enumerator { - /** How many moveNext() calls to perform a resource check once. */ - private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; - - private final List fields; - - @EqualsAndHashCode.Include @ToString.Include private final RestRequest request; - - private Iterator iterator; - - private ExprValue current; - - /** Number of rows returned. */ - private Integer queryCount; - - /** ResourceMonitor. */ - private final ResourceMonitor monitor; - - public RestEnumerator(List fields, RestRequest request, ResourceMonitor monitor) { - this.fields = fields; - this.request = request; - this.monitor = monitor; - this.queryCount = 0; - this.current = null; - if (!this.monitor.isHealthy()) { - throw new NonFallbackCalciteException("insufficient resources to run the query, quit."); - } - this.iterator = request.search().iterator(); - } - - @Override - public Object current() { - return fields.stream() - .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) - .toArray(); - } - - @Override - public boolean moveNext() { - boolean shouldCheck = (queryCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); - if (shouldCheck && !this.monitor.isHealthy()) { - throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); - } - if (iterator.hasNext()) { - current = iterator.next(); - queryCount++; - return true; - } else { - return false; - } - } - - @Override - public void reset() { - iterator = request.search().iterator(); - queryCount = 0; - current = null; - } - - @Override - public void close() { - iterator = null; - current = null; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java deleted file mode 100644 index a900a1d319e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Map; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on - * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the - * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only - * allow-list), exposes the endpoint's fixed schema, and produces a {@link CalciteLogicalRestScan} - * on the Calcite path. - */ -@Getter -public class RestSourceTable extends AbstractOpenSearchTable { - - private final OpenSearchClient client; - private final Settings settings; - private final RestSpec spec; - private final RestEndpointRegistry.Endpoint endpoint; - - public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec) { - this.client = client; - this.settings = settings; - this.spec = spec; - // Allow-list enforced here: unknown/mutating endpoints and disallowed args - // are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); - } - - @Override - public boolean exists() { - return true; - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException("rest endpoint is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return endpoint.getSchema(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalRestScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); - } - - public RestRequest createRestRequest() { - return new RestRequest(client, endpoint, spec); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 81% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index a55d29e785a..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.storage.rest; +package org.opensearch.sql.opensearch.storage.system; import java.util.List; import org.apache.calcite.adapter.enumerable.EnumerableConvention; @@ -25,23 +25,22 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -import org.opensearch.sql.calcite.plan.Scannable; -/** The physical relational operator representing a scan of a {@link RestSourceTable}. */ -public class CalciteEnumerableRestScan extends AbstractCalciteRestScan - implements EnumerableRel, Scannable { - public CalciteEnumerableRestScan( +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan + implements EnumerableRel { + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - RestSourceTable restTable, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, - restTable, + catalogTable, schema); } @@ -66,19 +65,18 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableRestScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } - @Override public Enumerable<@Nullable Object> scan() { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new RestEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - restTable.createRestRequest(), - restTable.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java deleted file mode 100644 index b0c92dce8f9..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.List; -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableRel; -import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; -import org.apache.calcite.adapter.enumerable.PhysType; -import org.apache.calcite.adapter.enumerable.PhysTypeImpl; -import org.apache.calcite.linq4j.AbstractEnumerable; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Enumerator; -import org.apache.calcite.linq4j.tree.Blocks; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.plan.DeriveMode; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan - implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( - RelOptCluster cluster, - List hints, - RelOptTable table, - OpenSearchSystemIndex sysIndex, - RelDataType schema) { - super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); - } - - @Override - public @Nullable Pair> passThroughTraits(RelTraitSet required) { - return EnumerableRel.super.passThroughTraits(required); - } - - @Override - public @Nullable Pair> deriveTraits( - RelTraitSet childTraits, int childId) { - return EnumerableRel.super.deriveTraits(childTraits, childId); - } - - @Override - public DeriveMode getDeriveMode() { - return EnumerableRel.super.getDeriveMode(); - } - - @Override - public Result implement(EnumerableRelImplementor implementor, Prefer pref) { - PhysType physType = - PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); - return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); - } - - public Enumerable<@Nullable Object> scan() { - return new AbstractEnumerable<>() { - @Override - public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( - getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); - } - }; - } - - private List getFieldPath() { - return getRowType().getFieldNames().stream().toList(); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..dff9b47265a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..d065e9ac0b6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -20,7 +20,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; @ExtendWith(MockitoExtension.class) @@ -52,6 +52,7 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java similarity index 60% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java index a50bf27f042..8676373d2a6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -15,7 +15,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import com.google.common.collect.ImmutableMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -23,47 +22,42 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; +/** + * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, + * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) + * path. + */ @ExtendWith(MockitoExtension.class) -class RestSourceTableTest { +class RestCatalogSourceTest { @Mock private OpenSearchClient client; - @Mock private Settings settings; - private RestSpec healthSpec() { return new RestSpec("/_cluster/health", Map.of(), null, null); } @Test void getFieldTypesReturnsFixedEndpointSchema() { - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - Map fieldTypes = table.getFieldTypes(); + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + Map fieldTypes = source.getFieldTypes(); assertThat(fieldTypes, hasEntry("status", STRING)); assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); } @Test - void existsIsTrue() { - Table table = new RestSourceTable(client, settings, healthSpec()); - assertTrue(table.exists()); - } - - @Test - void createIsUnsupported() { - Table table = new RestSourceTable(client, settings, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> table.create(ImmutableMap.of())); + void isScannable() { + assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); } @Test - void implementIsUnsupportedOnV2() { - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> table.implement(null)); + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); } @Test @@ -71,8 +65,7 @@ void constructorRejectsNonAllowListedEndpoint() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, settings, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); } @Test @@ -80,19 +73,15 @@ void constructorRejectsDisallowedArg() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, - settings, - new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + new RestCatalogSource( + client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); } @Test void constructorRejectsNegativeCount() { assertThrows( IllegalArgumentException.class, - () -> - new RestSourceTable( - client, settings, new RestSpec("/_cat/indices", Map.of(), -1, null))); + () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); } @Test @@ -100,8 +89,7 @@ void constructorRejectsTimeoutArg() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, settings, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); } @Test @@ -111,8 +99,8 @@ void restRequestShapesResponseRows() { health.put("number_of_nodes", 1); when(client.clusterHealth(any())).thenReturn(health); - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - List rows = table.createRestRequest().search(); + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + List rows = source.createRequest().search(); assertEquals(1, rows.size()); assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); @@ -126,9 +114,9 @@ void countTruncatesRows() { idx2.put("index", "b"); when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - RestSourceTable table = - new RestSourceTable(client, settings, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = table.createRestRequest().search(); + RestCatalogSource source = + new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = source.createRequest().search(); assertEquals(1, rows.size()); assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index f26a2e617e4..da9424c9c8f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -22,11 +22,11 @@ * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. * *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> - * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} - * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch - * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is - * exercised in {@code RestSourceTableTest} (logical scan + fixed row type, unit) and {@code - * CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides * {@code visitRelation} exactly like {@code DESCRIBE}. From 9afadf8202f7b60304441c85fd82a4c8ae6c5217 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 04:06:50 +0800 Subject: [PATCH 09/23] [Feature] Add rest command response redaction and endpoint allow-list Two dynamic cluster settings gate the rest command per deployment: - plugins.ppl.rest.redaction.enabled (default false): mask network identifiers in _cat/* cells and availability-zone names in _cluster/settings values. - plugins.ppl.rest.allowed_endpoints (default all): restrict which endpoints are served; an empty list disables the rest command. Both default to open-source parity: all endpoints served, no masking. Signed-off-by: Louis Chu --- .../sql/common/setting/Settings.java | 2 + .../setting/OpenSearchSettings.java | 29 ++++++ .../storage/OpenSearchStorageEngine.java | 20 +++- .../storage/rest/RestCatalogSource.java | 8 +- .../storage/rest/RestEndpointRegistry.java | 34 ++++++- .../opensearch/storage/rest/RestRequest.java | 12 ++- .../storage/rest/RestResponseRedactor.java | 61 ++++++++++++ .../storage/OpenSearchStorageEngineTest.java | 66 +++++++++++++ .../rest/RestEndpointRegistryTest.java | 76 +++++++++++++++ .../rest/RestResponseRedactorTest.java | 94 +++++++++++++++++++ 10 files changed, 397 insertions(+), 5 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 96fe2e04eea..9419b179bb8 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,8 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index bd8001f589d..902d9a55684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -70,6 +70,21 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = + Setting.boolSetting( + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of("*"), + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -371,6 +386,18 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.PPL_REST_REDACTION_ENABLED, + PPL_REST_REDACTION_ENABLED_SETTING, + new Updater(Key.PPL_REST_REDACTION_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING, + new Updater(Key.PPL_REST_ALLOWED_ENDPOINTS)); register( settingBuilder, clusterSettings, @@ -651,6 +678,8 @@ public static List> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(PPL_ENABLED_SETTING) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) .add(CALCITE_ENGINE_ENABLED_SETTING) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 64bc83211b1..a2afd9a9992 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -22,6 +22,7 @@ import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -40,12 +41,27 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { if (isRestSource(name)) { - return new OpenSearchCatalogTable( - new RestCatalogSource(client, decodeRestSpec(name)), settings); + return restTable(name); } else if (isSystemIndex(name)) { return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); + return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java index 807f83df4bf..96779726a7e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -26,10 +26,16 @@ public class RestCatalogSource implements CatalogSource { private final OpenSearchClient client; private final RestSpec spec; private final RestEndpointRegistry.Endpoint endpoint; + private final boolean redact; public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this(client, spec, false); + } + + public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { this.client = client; this.spec = spec; + this.redact = redact; // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); RestEndpointRegistry.validate(spec); @@ -42,7 +48,7 @@ public Map getFieldTypes() { @Override public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec); + return new RestRequest(client, endpoint, spec, redact); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index feb6a6fafe3..e64a91ab54a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -69,16 +69,48 @@ public static final class Endpoint { /** Dispatch the read-only call and shape the response into fixed-schema rows. */ public List toRows(OpenSearchClient client, RestSpec spec) { + return toRows(client, spec, false); + } + + /** + * Shape the response into fixed-schema rows, masking network identifiers when redaction is + * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value + * column is zone-masked. Off by default. + */ + public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { + boolean redactCat = redact && path.startsWith("/_cat"); + boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); List out = new ArrayList<>(); for (Map raw : fetcher.fetch(client, spec)) { LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { - tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); + ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); + tuple.put( + col.getKey(), + maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); } out.add(new ExprTupleValue(tuple)); } return out; } + + private static ExprValue maskCell( + String column, + ExprType type, + ExprValue value, + boolean redactCat, + boolean redactSettingsValue) { + if (type != STRING || value.isNull()) { + return value; + } + if (redactCat) { + return stringValue(RestResponseRedactor.redact(value.stringValue())); + } + if (redactSettingsValue && "value".equals(column)) { + return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + } + return value; + } } private static final Map REGISTRY = buildRegistry(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index d6ee4dff1ac..868dabdd64e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -23,17 +23,27 @@ public class RestRequest implements OpenSearchSystemRequest { private final OpenSearchClient client; private final RestEndpointRegistry.Endpoint endpoint; private final RestSpec spec; + private final boolean redact; public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this(client, endpoint, spec, false); + } + + public RestRequest( + OpenSearchClient client, + RestEndpointRegistry.Endpoint endpoint, + RestSpec spec, + boolean redact) { this.client = client; this.endpoint = endpoint; this.spec = spec; + this.redact = redact; } @Override public List search() { - List rows = endpoint.toRows(client, spec); + List rows = endpoint.toRows(client, spec, redact); if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { return rows.subList(0, spec.getCount()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java new file mode 100644 index 00000000000..89ba8d47244 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * Masks network identifiers in rest command cell values. Enabled per deployment via {@code + * plugins.ppl.rest.redaction.enabled}; off by default. + */ +public final class RestResponseRedactor { + + private RestResponseRedactor() {} + + private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; + private static final Pattern IPV4 = + Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); + private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); + private static final Pattern EC2_HOST = + Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); + private static final Pattern IPV6 = + Pattern.compile("([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}", Pattern.CASE_INSENSITIVE); + private static final Pattern AZ_NAME = + Pattern.compile( + "(us(-(gov|iso[a-z]?))?|af|ap|ca|cn|eu|sa|me|il)-(central|(north|south)?(east|west)?)-\\d[a-z]", + Pattern.CASE_INSENSITIVE); + + private record Mask(Pattern pattern, String replacement) {} + + private static final List MASKS = + List.of( + new Mask(IPV4, "x.x.x.x"), + new Mask(INET, "inet[/x.x.x.x:y]"), + new Mask(EC2_HOST, ""), + new Mask(IPV6, "x.x.x.x"), + new Mask(AZ_NAME, "xx-xxxxx-xx")); + + /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ + public static String redact(String text) { + if (text == null || text.isEmpty()) { + return text; + } + String out = text; + for (Mask mask : MASKS) { + out = mask.pattern().matcher(out).replaceAll(mask.replacement()); + } + return out; + } + + /** Mask availability-zone names only. */ + public static String maskAvailabilityZone(String text) { + if (text == null || text.isEmpty()) { + return text; + } + return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index d065e9ac0b6..102ec4da8f7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -22,6 +26,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -55,4 +60,65 @@ public void getSystemTable() { assertAll( () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); } + + @Test + public void getRestTableAllowedByWildcard() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + Table table = + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); + assertTrue(table instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index 3254ee12f09..c5a978bb495 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -59,6 +59,82 @@ void validateAcceptsAllowedArg() { RestEndpointRegistry.validate(spec); // no throw } + @Test + void catEndpointsRedactAddressesWhenRedactionEnabled() { + Map node = new LinkedHashMap<>(); + node.put("name", "ip-10-0-0-7"); + node.put("ip", "10.0.0.7"); + node.put("node_role", "dir"); + node.put("heap_percent", 44); + node.put("ram_percent", 95); + node.put("cpu", 2); + when(client.catNodes(any())).thenReturn(List.of(node)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); + RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("", redacted.get("name").stringValue()); + assertEquals(44, redacted.get("heap_percent").integerValue()); + + Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); + assertEquals("10.0.0.7", plain.get("ip").stringValue()); + assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + } + + @Test + void catClusterManagerRedactsHostAndIp() { + Map row = new LinkedHashMap<>(); + row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); + row.put("host", "10.0.0.7"); + row.put("ip", "10.0.0.7"); + row.put("node", "71d03b567bb755839a73d437b2b066d4"); + when(client.catClusterManager(any())).thenReturn(List.of(row)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); + RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("host").stringValue()); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); + assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + } + + @Test + void nonCatEndpointNotRedactedEvenWhenEnabled() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "10.0.0.7"); + health.put("status", "green"); + health.put("number_of_nodes", 3); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + } + + @Test + void clusterSettingsMasksAvailabilityZoneInValue() { + Map setting = new LinkedHashMap<>(); + setting.put("setting", "cluster.routing.allocation.awareness.attributes"); + setting.put("value", "zone:us-east-1a"); + setting.put("tier", "persistent"); + when(client.clusterSettings(any())).thenReturn(List.of(setting)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); + RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); + assertEquals( + "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); + assertEquals("persistent", row.get("tier").stringValue()); + } + @Test void validateRejectsDroppedLevelArg() { // level was dropped (no-op against the fixed cluster-level health schema); now unknown. diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java new file mode 100644 index 00000000000..81d60661f50 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class RestResponseRedactorTest { + + @Test + void masksIpv4() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); + assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); + assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); + } + + @Test + void doesNotMaskInvalidOrPartialIpv4() { + assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); + assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); + assertEquals("44", RestResponseRedactor.redact("44")); + } + + @Test + void masksEc2HostName() { + assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); + assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); + assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); + assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); + } + + @Test + void masksFullIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); + } + + @Test + void doesNotMaskCompressedIpv6() { + assertEquals("::1", RestResponseRedactor.redact("::1")); + assertEquals("2001:db8::1", RestResponseRedactor.redact("2001:db8::1")); + } + + @Test + void masksInetAddress() { + assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); + } + + @Test + void masksAvailabilityZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + assertEquals( + "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); + } + + @Test + void maskAvailabilityZoneMasksOnlyZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); + assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); + assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); + } + + @Test + void leavesNonAddressesIntact() { + assertEquals( + "e4e136ea81e27370ff73cf753ba22d39", + RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); + assertEquals( + "data,ingest,remote_cluster_client", + RestResponseRedactor.redact("data,ingest,remote_cluster_client")); + assertEquals( + "x.x.x.x 44 95 imr - e4e136ea", + RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); + } + + @Test + void handlesNullAndEmpty() { + assertEquals(null, RestResponseRedactor.redact(null)); + assertEquals("", RestResponseRedactor.redact("")); + assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); + assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); + } +} From f2aeb5f1678cb12ff2569677fd7d2d7c5d3d01c2 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 04:18:00 +0800 Subject: [PATCH 10/23] [Refactor] Remove unused REST/TIMEOUT rules from shared language grammar The shared language-grammar carried REST and TIMEOUT lexer tokens and the restCommand/restArgument parser rules with no consumer: async-query-core has no rest visitor, and the rest command grammar lives in the ppl module. Remove the dead rules. Signed-off-by: Louis Chu --- .../src/main/antlr4/OpenSearchPPLLexer.g4 | 2 -- .../src/main/antlr4/OpenSearchPPLParser.g4 | 11 ----------- 2 files changed, 13 deletions(-) diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index 8ba747614da..2248374d8d9 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,8 +13,6 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; -REST: 'REST'; -TIMEOUT: 'TIMEOUT'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index 1d13e8af046..451824d6992 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -62,7 +62,6 @@ commands commandName : SEARCH | DESCRIBE - | REST | SHOW | AD | ML @@ -114,16 +113,6 @@ describeCommand : DESCRIBE tableSourceClause ; - -restCommand - : REST stringLiteral (restArgument)* - ; - -restArgument - : COUNT EQUAL integerLiteral - | TIMEOUT EQUAL stringLiteral - | ident EQUAL literalValue - ; explainCommand : EXPLAIN explainMode ; From b0eff8e808df7ac05e36525180c32bb12f5a5c73 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 12:32:07 +0800 Subject: [PATCH 11/23] [Test] Add rest command security integration tests Verify the rest command is subject to the security plugin fine grained access control: a caller without cluster:monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege can run them, and the resolve index endpoint is filtered to the caller authorized indices. Test indices are created idempotently, and denials are asserted by the security denial reason in the response body because a denied action on the Calcite only rest path surfaces as a wrapped error carrying that reason. Calcite fallback is disabled so the denial reason is not replaced by an unsupported command error. Signed-off-by: Louis Chu --- .../sql/security/RestCommandSecurityIT.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..042a3aefbf7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that verify the rest command is subject to the security plugin fine grained + * access control. The command dispatches standard transport actions under the caller identity, so + * the security ActionFilter authorizes each one by action name. A caller without the required + * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege + * can run them, and the resolve index endpoint requires the resolve index privilege because the + * command resolves all indices. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String ALPHA_INDEX = "rest_sec_alpha"; + private static final String BETA_INDEX = "rest_sec_beta"; + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesUsersAndIndices(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesUsersAndIndices() throws IOException { + createIndexIfAbsent(ALPHA_INDEX); + createIndexIfAbsent(BETA_INDEX); + + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] { + "cluster:admin/opensearch/ppl", + "cluster:monitor/health", + "cluster:monitor/state", + "cluster:monitor/nodes/stats", + "cluster:monitor/nodes/info" + }, + new String[] {"indices:admin/resolve/index"}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, + ALPHA_INDEX, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] {"indices:data/read/search*"}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunCatNodes() throws IOException { + JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); + verifyColumn(result, columnName("name")); + } + + @Test + public void monitorUserCanResolveIndex() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); + Set names = resolvedNames(result); + assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); + assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); + } + + @Test + public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { + assertDenied( + "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { + assertDenied( + "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + } + + @Test + public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + assertDenied( + "| rest '/_resolve/index' | fields name, type", + NO_MONITOR_USER, + "indices:admin/resolve/index"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } + + private void createIndexIfAbsent(String name) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity( + "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); + try { + client().performRequest(request); + } catch (ResponseException e) { + String body = TestUtils.getResponseBody(e.getResponse(), false); + if (!body.contains("resource_already_exists_exception")) { + throw e; + } + } + } + + private Set resolvedNames(JSONObject result) { + Set names = new HashSet<>(); + JSONArray datarows = result.getJSONArray("datarows"); + for (int i = 0; i < datarows.length(); i++) { + names.add(datarows.getJSONArray(i).getString(0)); + } + return names; + } +} From 0ed1f36bceb32e93de8f88dc020b0f834a20d335 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Fri, 10 Jul 2026 09:50:13 +0800 Subject: [PATCH 12/23] [Bugfix] Make rest redaction and allow-list settings node-level plugins.ppl.rest.redaction.enabled and plugins.ppl.rest.allowed_endpoints were dynamic cluster settings, so they could be changed at runtime through _cluster/settings or the _plugins/_query/settings endpoint, which could override an operator-configured value. Drop Setting.Property.Dynamic so both are node-level settings set in the node config; the engine rejects runtime updates on both paths. Register them without an update consumer and read the node-configured value. Signed-off-by: Louis Chu --- .../setting/OpenSearchSettings.java | 26 ++++++++----------- .../setting/OpenSearchSettingsTest.java | 12 +++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 902d9a55684..13af16a297f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -72,18 +72,14 @@ public class OpenSearchSettings extends Settings { public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), - false, - Setting.Property.NodeScope, - Setting.Property.Dynamic); + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), List.of("*"), Function.identity(), - Setting.Property.NodeScope, - Setting.Property.Dynamic); + Setting.Property.NodeScope); public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( @@ -386,18 +382,16 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - register( + registerNonDynamicSettings( settingBuilder, clusterSettings, Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING, - new Updater(Key.PPL_REST_REDACTION_ENABLED)); - register( + PPL_REST_REDACTION_ENABLED_SETTING); + registerNonDynamicSettings( settingBuilder, clusterSettings, Key.PPL_REST_ALLOWED_ENDPOINTS, - PPL_REST_ALLOWED_ENDPOINTS_SETTING, - new Updater(Key.PPL_REST_ALLOWED_ENDPOINTS)); + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -652,7 +646,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -678,8 +674,6 @@ public static List> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(PPL_ENABLED_SETTING) - .add(PPL_REST_REDACTION_ENABLED_SETTING) - .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) .add(CALCITE_ENGINE_ENABLED_SETTING) @@ -724,6 +718,8 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..0c570098924 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +77,15 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); From dab07e92999506c3afe5fd1d373272cf1a586fb0 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 15 Jul 2026 03:57:02 +0800 Subject: [PATCH 13/23] Enhance redaction logic Signed-off-by: Louis Chu --- .../opensearch/storage/rest/RestResponseRedactor.java | 7 +++++-- .../storage/rest/RestResponseRedactorTest.java | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java index 89ba8d47244..fd674dccde4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -23,10 +23,13 @@ private RestResponseRedactor() {} private static final Pattern EC2_HOST = Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); private static final Pattern IPV6 = - Pattern.compile("([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}", Pattern.CASE_INSENSITIVE); + Pattern.compile( + "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" + + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", + Pattern.CASE_INSENSITIVE); private static final Pattern AZ_NAME = Pattern.compile( - "(us(-(gov|iso[a-z]?))?|af|ap|ca|cn|eu|sa|me|il)-(central|(north|south)?(east|west)?)-\\d[a-z]", + "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", Pattern.CASE_INSENSITIVE); private record Mask(Pattern pattern, String replacement) {} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java index 81d60661f50..3b038306da5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -44,9 +44,11 @@ void masksFullIpv6() { } @Test - void doesNotMaskCompressedIpv6() { - assertEquals("::1", RestResponseRedactor.redact("::1")); - assertEquals("2001:db8::1", RestResponseRedactor.redact("2001:db8::1")); + void masksCompressedIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); } @Test @@ -60,6 +62,8 @@ void masksAvailabilityZones() { assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); assertEquals( "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); } From 387ed50b659c40b97a73c9f30f23f20e144befca Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 15 Jul 2026 10:34:52 +0800 Subject: [PATCH 14/23] [Change] Disable all rest endpoints by default (empty allow-list) Flip plugins.ppl.rest.allowed_endpoints default from ["*"] to empty so open source ships with the rest command closed. Deployments opt specific endpoints in via the setting (AOS enables the ones it supports; AOSS leaves it empty and stays disabled). Enforcement already treats an empty or missing list as disabled. Opt the integration-test clusters into all endpoints so the rest ITs still exercise the enabled path. Signed-off-by: Louis Chu --- docs/user/ppl/cmd/rest.md | 12 ++++++++++++ doctest/build.gradle | 5 +++++ integ-test/build.gradle | 8 ++++++++ .../sql/opensearch/setting/OpenSearchSettings.java | 2 +- .../opensearch/storage/OpenSearchStorageEngine.java | 2 ++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index b97f247445f..a9761aa87e8 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -4,6 +4,18 @@ The `rest` command is a leading command that reads an allow-listed, read-only in > **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. +## Enabling the command + +The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): + +```yaml +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +``` + +Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. + +The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. + ## Syntax The `rest` command has the following syntax: diff --git a/doctest/build.gradle b/doctest/build.gradle index cce64170f56..1ac658457dc 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,6 +205,11 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' + // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into + // the registered endpoints so the rest.md examples run against an enabled command. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d04c4484cc1..a377d343c1b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,6 +387,11 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + // The rest command is disabled by default (empty allow-list). Opt this cluster into the + // registered endpoints so the rest integration tests exercise the enabled path. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -405,6 +410,9 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" + // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 13af16a297f..55212248b43 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -77,7 +77,7 @@ public class OpenSearchSettings extends Settings { public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of("*"), + List.of(), Function.identity(), Setting.Property.NodeScope); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index a2afd9a9992..7b911471242 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -18,6 +18,7 @@ import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; @@ -51,6 +52,7 @@ public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { private Table restTable(String name) { RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry.resolve(spec.getEndpoint()); List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { throw new IllegalArgumentException( From 591fd8b7a4838aa852c6b7099be3712d1d4482ac Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Mon, 27 Jul 2026 22:28:44 +0800 Subject: [PATCH 15/23] Add a generic, extensible rest-endpoint provider SPI (with a /_cluster/health sample) Rebuild of the reverted PPL rest command as a generic framework. A RestEndpointProvider ExtensiblePlugin SPI (in a thin ppl-rest-spi module) lets any plugin contribute read-only, fixed-schema endpoints resolved through one allow-list choke point and run as a lazy relational scan; the grammar stays a single generic rest command. Ships one built-in endpoint /_cluster/health as the reference provider, allow-listed by default; any other endpoint stays disabled until explicitly added to plugins.ppl.rest.allowed_endpoints. Redaction is applied centrally at the choke point rather than through the endpoint SPI. Additional endpoints are left to follow-up changes. Signed-off-by: Louis Chu --- .../sql/common/setting/Settings.java | 1 - docs/user/ppl/cmd/rest.md | 56 +-- .../sql/calcite/remote/CalcitePPLRestIT.java | 168 ++------- .../sql/security/RestCommandSecurityIT.java | 96 +---- opensearch/build.gradle | 1 + .../opensearch/client/OpenSearchClient.java | 101 ------ .../client/OpenSearchNodeClient.java | 273 -------------- .../client/OpenSearchRestClient.java | 268 -------------- .../setting/OpenSearchSettings.java | 12 +- .../storage/OpenSearchStorageEngine.java | 17 +- .../storage/rest/CoreEndpointsProvider.java | 87 +++++ .../storage/rest/RedactionRegistry.java | 73 ++++ .../storage/rest/RedactionRegistryHolder.java | 31 ++ .../storage/rest/RestCatalogSource.java | 24 +- .../storage/rest/RestEndpointRegistry.java | 343 +++++------------- .../rest/RestEndpointRegistryHolder.java | 32 ++ .../opensearch/storage/rest/RestRequest.java | 22 +- .../storage/rest/RestResponseRedactor.java | 64 ---- .../rest/RestSettingsFilterHolder.java | 40 -- ...chNodeClientClusterSettingsFilterTest.java | 99 ----- .../setting/OpenSearchSettingsTest.java | 3 - .../storage/OpenSearchStorageEngineTest.java | 26 +- .../rest/CoreEndpointsProviderTest.java | 63 ++++ .../storage/rest/RedactionByClassTest.java | 101 ++++++ .../storage/rest/RestCatalogSourceTest.java | 82 +++-- .../rest/RestEndpointExtensibilityTest.java | 102 ++++++ .../rest/RestEndpointRegistryTest.java | 278 +++++--------- .../rest/RestResponseRedactorTest.java | 98 ----- .../org/opensearch/sql/plugin/SQLPlugin.java | 25 +- ppl-rest-spi/build.gradle | 60 +++ .../org/opensearch/sql/spi/rest/ArgSpec.java | 111 ++++++ .../org/opensearch/sql/spi/rest/Column.java | 31 ++ .../opensearch/sql/spi/rest/ColumnType.java | 21 ++ .../sql/spi/rest/RedactionClass.java | 28 ++ .../org/opensearch/sql/spi/rest/Redactor.java | 25 ++ .../sql/spi/rest/RestEndpointContext.java | 47 +++ .../sql/spi/rest/RestEndpointDefinition.java | 89 +++++ .../sql/spi/rest/RestEndpointHandler.java | 33 ++ .../sql/spi/rest/RestEndpointProvider.java | 25 ++ .../sql/ppl/calcite/CalcitePPLRestTest.java | 5 +- settings.gradle | 1 + 41 files changed, 1341 insertions(+), 1721 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java create mode 100644 ppl-rest-spi/build.gradle create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 9419b179bb8..ec0f1f67191 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,7 +36,6 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), - PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index a9761aa87e8..e2cf098fa7d 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -1,39 +1,34 @@ # rest -The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. + +The `rest` command is a generic, extensible framework: a plugin contributes additional read-only endpoints through the `RestEndpointProvider` extension point without changing the grammar. This first version ships a single built-in endpoint, `/_cluster/health`. Endpoints that surface network topology or other sensitive metadata (for example `/_cat/nodes`, `/_cat/shards`, `/_cluster/state`, `/_cluster/settings`) are gated behind a security review and are added subsequently, together with schema-declared response redaction (a `RedactionClass` per column, masked by a platform-owned redactor). ## Enabling the command -The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): +`/_cluster/health` is **enabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to `["/_cluster/health"]`. Any other endpoint is rejected until a deployment adds it to the allow-list (a node-level setting, applied at node startup and not changeable at runtime): ```yaml -plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health"] ``` -Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. - -The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. +Use `["*"]` to allow every endpoint in the curated list below. Set an empty list to disable the command entirely. ## Syntax -The `rest` command has the following syntax: - ```syntax -rest [count=] [timeout=] [= ...] +rest [count=] [= ...] ``` ## Parameters -The `rest` command supports the following parameters. - | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | | `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | -| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`). | ## Allow-list @@ -42,16 +37,8 @@ The `rest` command supports the following parameters. | Endpoint | Output columns | Accepted args | | --- | --- | --- | | `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | -| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | -| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | -| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | -| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | -| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | -| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | -| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | -## Example 1: Counting the nodes in the cluster +## Example: Counting the nodes in the cluster The following query reads cluster health and projects a column that is deterministic on a single-node cluster: @@ -70,25 +57,4 @@ fetched rows / total rows = 1/1 +-----------------+ ``` -`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. - -## Example 2: Composing downstream commands over a cat endpoint - -The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: - -```ppl -| rest '/_cat/cluster_manager' | stats count() as managers -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+----------+ -| managers | -|----------| -| 1 | -+----------+ -``` - -For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. +`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list. The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan, for example `| rest '/_cluster/health' | where status = 'green' | fields status, active_shards`. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java index 70ee4ef0de6..e779e6c8c12 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -13,14 +13,14 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also - * verifies that a non-allow-listed / mutating endpoint is refused. + * Integration tests for the {@code rest} leading command on the Calcite path. This first version + * ships a single built-in endpoint, {@code /_cluster/health} (a deterministic single-row endpoint + * that carries no network identifiers and needs no redaction). These tests exercise it end to end + * and verify the allow-list and per-arg gates. */ public class CalcitePPLRestIT extends PPLIntegTestCase { @@ -38,34 +38,45 @@ public void testRestClusterHealthSchema() throws IOException { @Test public void testRestClusterHealthDataRows() throws IOException { - // Single-node test cluster: exactly one node, status is green or yellow. + // Single-node test cluster: exactly one node. JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); verifyDataRows(result, rows(1)); } @Test - public void testRestRejectsNonAllowListedEndpoint() throws IOException { - assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + public void testRestClusterHealthComposesDownstream() throws IOException { + // The rest row source composes with downstream stats exactly like an index scan. + JSONObject result = executeQuery("| rest '/_cluster/health' | stats count() as cnt"); + verifyDataRows(result, rows(1)); } @Test - public void testRestRejectsEmptyEndpoint() throws IOException { - assertRestBadRequest("| rest ''", "non-empty path"); + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() { + // /_cat/nodes is not registered in this version; it is refused before any transport call. + assertRestBadRequest("| rest '/_cat/nodes'", "allow-list"); } @Test - public void testRestRejectsDisallowedArg() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + public void testRestRejectsEmptyEndpoint() { + assertRestBadRequest("| rest ''", "non-empty path"); } @Test - public void testRestRejectsNegativeCount() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + public void testRestRejectsDisallowedArg() { + assertRestBadRequest("| rest '/_cluster/health' h='name'", "does not accept arg"); } @Test - public void testRestRejectsTimeoutArg() throws IOException { - assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + public void testRestRejectsNegativeCount() { + assertRestBadRequest("| rest '/_cluster/health' count=-1", "non-negative"); } /** @@ -80,131 +91,4 @@ private void assertRestBadRequest(String query, String expectedSubstring) { "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), e.getMessage().contains(expectedSubstring)); } - - @Test - public void testRestCatIndicesSchema() throws IOException { - // Schema is fixed by the registry, independent of how many indices exist. - JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); - verifySchema(result, schema("index", "string"), schema("health", "string")); - } - - @Test - public void testRestCatIndicesReturnsCreatedIndex() throws IOException { - // Create a known index, then confirm rest surfaces it and downstream where/fields compose. - client().performRequest(new Request("PUT", "/rest_cat_test")); - JSONObject result = - executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); - verifyDataRows(result, rows("rest_cat_test")); - } - - @Test - public void testRestCatNodesSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); - verifySchema(result, schema("name", "string"), schema("cpu", "int")); - } - - @Test - public void testRestCatNodesSingleNode() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatClusterManagerSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); - verifySchema(result, schema("node", "string"), schema("id", "string")); - } - - @Test - public void testRestCatClusterManagerSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatPluginsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); - verifySchema(result, schema("component", "string"), schema("version", "string")); - } - - @Test - public void testRestCatShardsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); - verifySchema( - result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); - } - - @Test - public void testRestClusterStateSchema() throws IOException { - // Assert the string columns; version is the LONG epoch column. - JSONObject result = - executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); - verifySchema( - result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); - } - - @Test - public void testRestClusterStateSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestClusterSettingsSchema() throws IOException { - // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); - verifySchema( - result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); - } - - @Test - public void testRestResolveIndexSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { - client().performRequest(new Request("PUT", "/rest_resolve_test")); - JSONObject result = - executeQuery( - "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); - verifyDataRows(result, rows("rest_resolve_test", "index")); - } - - // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- - - @Test - public void testRestClusterHealthLocalArg() throws IOException { - // local=true reads health from the local node; on a single-node cluster the row is unchanged. - JSONObject result = - executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { - // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); - verifyDataRows(result, rows(0)); - } - - @Test - public void testRestResolveIndexExpandWildcardsArg() throws IOException { - // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. - JSONObject result = - executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestRejectsDroppedLevelArg() throws IOException { - // level was dropped from the allow-list (no-op against the fixed health schema). - assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); - } - - @Test - public void testRestRejectsBadArgValue() throws IOException { - assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java index 042a3aefbf7..091846097ca 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -9,28 +9,21 @@ import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; -import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.TestUtils; /** - * Integration tests that verify the rest command is subject to the security plugin fine grained - * access control. The command dispatches standard transport actions under the caller identity, so - * the security ActionFilter authorizes each one by action name. A caller without the required - * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege - * can run them, and the resolve index endpoint requires the resolve index privilege because the - * command resolves all indices. + * Integration tests that the rest command is subject to the security plugin fine grained access + * control. The command dispatches a standard transport action under the caller identity, so the + * security ActionFilter authorizes it by action name. This version ships only {@code + * /_cluster/health}, which requires the {@code cluster:monitor/health} privilege: a caller holding + * cluster monitor can run it, a caller without it is denied. The command therefore grants no access + * beyond calling the endpoint natively. */ public class RestCommandSecurityIT extends SecurityTestBase { - private static final String ALPHA_INDEX = "rest_sec_alpha"; - private static final String BETA_INDEX = "rest_sec_beta"; - private static final String MONITOR_USER = "rest_monitor_user"; private static final String MONITOR_ROLE = "rest_monitor_role"; @@ -40,71 +33,37 @@ public class RestCommandSecurityIT extends SecurityTestBase { @Override protected void init() throws Exception { super.init(); - setupRolesUsersAndIndices(); + setupRolesAndUsers(); enableCalcite(); // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported // command error. Disable fallback so the denial reason surfaces to the caller. disallowCalciteFallback(); } - private void setupRolesUsersAndIndices() throws IOException { - createIndexIfAbsent(ALPHA_INDEX); - createIndexIfAbsent(BETA_INDEX); - + private void setupRolesAndUsers() throws IOException { createRoleWithPermissions( MONITOR_ROLE, "*", - new String[] { - "cluster:admin/opensearch/ppl", - "cluster:monitor/health", - "cluster:monitor/state", - "cluster:monitor/nodes/stats", - "cluster:monitor/nodes/info" - }, - new String[] {"indices:admin/resolve/index"}); + new String[] {"cluster:admin/opensearch/ppl", "cluster:monitor/health"}, + new String[] {}); createUser(MONITOR_USER, MONITOR_ROLE); createRoleWithPermissions( - NO_MONITOR_ROLE, - ALPHA_INDEX, - new String[] {"cluster:admin/opensearch/ppl"}, - new String[] {"indices:data/read/search*"}); + NO_MONITOR_ROLE, "*", new String[] {"cluster:admin/opensearch/ppl"}, new String[] {}); createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); } @Test - public void monitorUserCanRunCatNodes() throws IOException { - JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); - verifyColumn(result, columnName("name")); - } - - @Test - public void monitorUserCanResolveIndex() throws IOException { + public void monitorUserCanRunClusterHealth() throws IOException { JSONObject result = - executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); - Set names = resolvedNames(result); - assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); - assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); - } - - @Test - public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { - assertDenied( - "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); - } - - @Test - public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { - assertDenied( - "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + executeQueryAsUser("| rest '/_cluster/health' | fields status", MONITOR_USER); + verifyColumn(result, columnName("status")); } @Test - public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + public void userWithoutClusterMonitorCannotRunClusterHealth() throws IOException { assertDenied( - "| rest '/_resolve/index' | fields name, type", - NO_MONITOR_USER, - "indices:admin/resolve/index"); + "| rest '/_cluster/health' | fields status", NO_MONITOR_USER, "cluster:monitor/health"); } /** @@ -128,27 +87,4 @@ private void assertDenied(String query, String user, String deniedAction) throws || body.contains(deniedAction)); } } - - private void createIndexIfAbsent(String name) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity( - "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); - try { - client().performRequest(request); - } catch (ResponseException e) { - String body = TestUtils.getResponseBody(e.getResponse(), false); - if (!body.contains("resource_already_exists_exception")) { - throw e; - } - } - } - - private Set resolvedNames(JSONObject result) { - Set names = new HashSet<>(); - JSONArray datarows = result.getJSONArray("datarows"); - for (int i = 0; i < datarows.length(); i++) { - names.add(datarows.getJSONArray(i).getString(0)); - } - return names; - } } diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 5ca10c7091c..03d16cad083 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,6 +32,7 @@ plugins { dependencies { api project(':core') + api project(':ppl-rest-spi') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:${resilience4j_version}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 3b9c3619521..68350c5a0fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,105 +114,4 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); - - /** - * Read-only cluster health snapshot for the {@code rest} command (backs {@code - * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's - * security thread-context; performs no privilege escalation and mutates nothing. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of health field name to value - */ - default Map clusterHealth(Map params) { - throw new UnsupportedOperationException("clusterHealth is not supported by this client"); - } - - /** - * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One - * map per index. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per index - */ - default List> catIndices(Map params) { - throw new UnsupportedOperationException("catIndices is not supported by this client"); - } - - /** - * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map - * per node with resource state. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per node - */ - default List> catNodes(Map params) { - throw new UnsupportedOperationException("catNodes is not supported by this client"); - } - - /** - * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code - * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map describing the cluster manager node - */ - default List> catClusterManager(Map params) { - throw new UnsupportedOperationException("catClusterManager is not supported by this client"); - } - - /** - * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One - * map per installed plugin per node. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per plugin - */ - default List> catPlugins(Map params) { - throw new UnsupportedOperationException("catPlugins is not supported by this client"); - } - - /** - * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map - * per shard. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per shard - */ - default List> catShards(Map params) { - throw new UnsupportedOperationException("catShards is not supported by this client"); - } - - /** - * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code - * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, - * cluster_manager_node). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of cluster-state field name to value - */ - default Map clusterState(Map params) { - throw new UnsupportedOperationException("clusterState is not supported by this client"); - } - - /** - * Read-only cluster-settings listing for the {@code rest} command (backs {@code - * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per setting - */ - default List> clusterSettings(Map params) { - throw new UnsupportedOperationException("clusterSettings is not supported by this client"); - } - - /** - * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). - * One map per resolved index, alias, or data stream (name, type). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per resolved name - */ - default List> resolveIndex(Map params) { - throw new UnsupportedOperationException("resolveIndex is not supported by this client"); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 080a8627894..b491f38ef80 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,8 +18,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -27,7 +25,6 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -288,274 +285,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); - return flattenHealth(response); - } - - @Override - public List> catIndices(Map params) { - ClusterHealthResponse response = - client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new java.util.LinkedHashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } - - @Override - public List> catNodes(Map params) { - org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = - new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); - statsRequest.all(); - org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = - client.admin().cluster().nodesStats(statsRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { - org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); - Map row = new java.util.LinkedHashMap<>(); - row.put("name", node.getName()); - row.put("ip", node.getHostAddress()); - row.put( - "node_role", - node.getRoles().stream() - .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) - .sorted() - .collect(java.util.stream.Collectors.joining(","))); - row.put( - "heap_percent", - ns.getJvm() == null || ns.getJvm().getMem() == null - ? null - : (int) ns.getJvm().getMem().getHeapUsedPercent()); - row.put( - "ram_percent", - ns.getOs() == null || ns.getOs().getMem() == null - ? null - : (int) ns.getOs().getMem().getUsedPercent()); - row.put( - "cpu", - ns.getProcess() == null || ns.getProcess().getCpu() == null - ? null - : (int) ns.getProcess().getCpu().getPercent()); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - List> rows = new java.util.ArrayList<>(); - if (cm != null) { - Map row = new java.util.LinkedHashMap<>(); - row.put("id", cm.getId()); - row.put("host", cm.getHostName()); - row.put("ip", cm.getHostAddress()); - row.put("node", cm.getName()); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = - new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); - infoRequest.all(); - org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = - client.admin().cluster().nodesInfo(infoRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { - org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = - info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); - if (plugins == null) { - continue; - } - for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", info.getNode().getName()); - row.put("component", pi.getName()); - row.put("version", pi.getVersion()); - rows.add(row); - } - } - return rows; - } - - @Override - public List> catShards(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.cluster.routing.ShardRouting sr : - response.getState().getRoutingTable().allShards()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("index", sr.getIndexName()); - row.put("shard", sr.id()); - row.put("prirep", sr.primary() ? "p" : "r"); - row.put("state", sr.state().name()); - org.opensearch.cluster.node.DiscoveryNode n = - sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); - row.put("node", n == null ? null : n.getName()); - rows.add(row); - } - return rows; - } - - @Override - public Map clusterState(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName().value()); - row.put("state_uuid", response.getState().stateUUID()); - row.put("version", response.getState().version()); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - row.put("cluster_manager_node", cm == null ? null : cm.getName()); - return row; - } - - @Override - public List> clusterSettings(Map params) { - // The transport path has no SettingsFilter of its own; it is published from - // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings - // into memory when we cannot redact them, matching native GET /_cluster/settings. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - org.opensearch.common.settings.Settings persistent = - filter.filter(response.getState().metadata().persistentSettings()); - org.opensearch.common.settings.Settings transientSettings = - filter.filter(response.getState().metadata().transientSettings()); - collectSettings(persistent, "persistent", rows); - collectSettings(transientSettings, "transient", rows); - return rows; - } - - private void collectSettings( - org.opensearch.common.settings.Settings settings, - String tier, - List> rows) { - if (settings == null) { - return; - } - for (String key : settings.keySet()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("setting", key); - String value = settings.get(key); - if (value == null) { - // List-valued settings return null from get(); fall back to the joined list form. - java.util.List list = settings.getAsList(key); - value = list.isEmpty() ? null : String.join(",", list); - } - row.put("value", value); - row.put("tier", tier); - rows.add(row); - } - } - - @Override - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = - expandWildcards == null - ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}) - : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}, - org.opensearch.action.support.IndicesOptions.fromParameters( - expandWildcards, - null, - null, - null, - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request - .DEFAULT_INDICES_OPTIONS)); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = - client - .execute( - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : - response.getIndices()) { - rows.add(resolveRow(idx.getName(), "index")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : - response.getAliases()) { - rows.add(resolveRow(alias.getName(), "alias")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : - response.getDataStreams()) { - rows.add(resolveRow(ds.getName(), "data_stream")); - } - return rows; - } - - private Map resolveRow(String name, String type) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", name); - row.put("type", type); - return row; - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index e98c5bf95f4..f369c0003b8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,19 +8,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -32,7 +28,6 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -277,267 +272,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - try { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); - return flattenHealth(response); - } catch (IOException e) { - throw new IllegalStateException("Failed to get cluster health", e); - } - } - - @Override - public List> catIndices(Map params) { - try { - ClusterHealthResponse response = - client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - List> rows = new ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new HashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } catch (IOException e) { - throw new IllegalStateException("Failed to get cat indices", e); - } - } - - @Override - public List> catNodes(Map params) { - List> raw = - catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("ip", r.get("ip")); - row.put("node_role", r.get("node.role")); - row.put("heap_percent", asInt(r.get("heap.percent"))); - row.put("ram_percent", asInt(r.get("ram.percent"))); - row.put("cpu", asInt(r.get("cpu"))); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("id", r.get("id")); - row.put("host", r.get("host")); - row.put("ip", r.get("ip")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - List> raw = catJson("/_cat/plugins", "name,component,version"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("component", r.get("component")); - row.put("version", r.get("version")); - rows.add(row); - } - return rows; - } - - @Override - public List> catShards(Map params) { - List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("index", r.get("index")); - row.put("shard", asInt(r.get("shard"))); - row.put("prirep", r.get("prirep")); - row.put("state", r.get("state")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ - @SuppressWarnings("unchecked") - private List> catJson(String path, String columns) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - request.addParameter("format", "json"); - request.addParameter("h", columns); - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - List list = parser.list(); - List> rows = new ArrayList<>(); - for (Object o : list) { - rows.add((Map) o); - } - return rows; - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Integer asInt(Object value) { - if (value == null) { - return null; - } - try { - return (int) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public Map clusterState(Map params) { - Map state = - getJsonMap( - "/_cluster/state/master_node,version,metadata,nodes", - // nodes.*.name resolves the manager id to a name without over-fetching node IPs. - Map.of( - "filter_path", - "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); - Map row = new HashMap<>(); - row.put("cluster_name", state.get("cluster_name")); - row.put("state_uuid", state.get("state_uuid")); - row.put("version", asLong(state.get("version"))); - Object cmId = state.get("cluster_manager_node"); - String cmName = null; - Object nodes = state.get("nodes"); - if (cmId != null && nodes instanceof Map) { - Object n = ((Map) nodes).get(cmId.toString()); - if (n instanceof Map) { - Object name = ((Map) n).get("name"); - cmName = name == null ? null : name.toString(); - } - } - row.put("cluster_manager_node", cmName); - return row; - } - - @Override - @SuppressWarnings("unchecked") - public List> clusterSettings(Map params) { - Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); - List> rows = new ArrayList<>(); - for (String tier : new String[] {"persistent", "transient"}) { - Object section = body.get(tier); - if (section instanceof Map) { - for (Map.Entry e : ((Map) section).entrySet()) { - Map row = new HashMap<>(); - row.put("setting", e.getKey()); - row.put("value", e.getValue() == null ? null : e.getValue().toString()); - row.put("tier", tier); - rows.add(row); - } - } - } - return rows; - } - - /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ - @SuppressWarnings("unchecked") - private Map getJsonMap(String path, Map params) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - if (params != null) { - params.forEach(request::addParameter); - } - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - return parser.map(); - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Long asLong(Object value) { - if (value == null) { - return null; - } - try { - return (long) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - Map body = - getJsonMap( - "/_resolve/index/*", - expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); - List> rows = new ArrayList<>(); - addResolved(body.get("indices"), "index", rows); - addResolved(body.get("aliases"), "alias", rows); - addResolved(body.get("data_streams"), "data_stream", rows); - return rows; - } - - @SuppressWarnings("unchecked") - private void addResolved(Object section, String type, List> rows) { - if (section instanceof List) { - for (Object o : (List) section) { - if (o instanceof Map) { - Map row = new HashMap<>(); - row.put("name", ((Map) o).get("name")); - row.put("type", type); - rows.add(row); - } - } - } - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new HashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 55212248b43..280399ed670 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -70,14 +70,10 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = - Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); - public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of(), + List.of("/_cluster/health"), Function.identity(), Setting.Property.NodeScope); @@ -382,11 +378,6 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING); registerNonDynamicSettings( settingBuilder, clusterSettings, @@ -718,7 +709,6 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) - .add(PPL_REST_REDACTION_ENABLED_SETTING) .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 7b911471242..31b36aed17f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -17,8 +17,11 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.storage.rest.RedactionRegistry; +import org.opensearch.sql.opensearch.storage.rest.RedactionRegistryHolder; import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; @@ -52,7 +55,12 @@ public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { private Table restTable(String name) { RestSpec spec = decodeRestSpec(name); - RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry registry = RestEndpointRegistryHolder.get(); + if (registry == null) { + throw new IllegalStateException( + "the rest command endpoint registry is not initialized on this node"); + } + registry.resolve(spec.getEndpoint()); List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { throw new IllegalArgumentException( @@ -63,7 +71,10 @@ private Table restTable(String name) { + "] is not enabled on this cluster. Enabled endpoints: " + allowed); } - boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); - return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + // Redaction is a platform-owned control applied by RedactionClass at the row-shaping choke + // point. OSS publishes an empty registry (no-op); a managed distribution fills it via patch. + RedactionRegistry redaction = RedactionRegistryHolder.get(); + return new OpenSearchCatalogTable( + new RestCatalogSource(registry, spec, client, redaction), settings); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java new file mode 100644 index 00000000000..7bfb2f21fe1 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.spi.rest.ColumnType.BOOLEAN; +import static org.opensearch.sql.spi.rest.ColumnType.INTEGER; +import static org.opensearch.sql.spi.rest.ColumnType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.transport.client.node.NodeClient; + +/** + * The built-in {@link RestEndpointProvider}. PR1 ships a single read-only, in-cluster endpoint, + * {@code /_cluster/health}, expressed as a {@link RestEndpointDefinition}. It is a uniform client + * of the same SPI an external plugin uses; it holds no privileged position in {@link + * RestEndpointRegistry}. The network-bearing endpoints ({@code /_cat/*}, {@code /_cluster/state}, + * {@code /_cluster/settings}, {@code /_resolve/index}, proposed {@code /_nodes/info}) are deferred + * to the AppSec follow-up. + * + *

Like any external provider, it fetches at execution time through the transport node client the + * context carries ({@link RestEndpointContext#client()}), so it holds no reference to the sql + * storage client and runs under the caller's security thread-context. + */ +public final class CoreEndpointsProvider implements RestEndpointProvider { + + @Override + public List getEndpoints() { + return List.of( + // /_cluster/health carries no network identifiers, so every column is RedactionClass.NONE. + RestEndpointDefinition.builder() + .name("/_cluster/health") + .schema( + List.of( + Column.of("cluster_name", STRING), + Column.of("status", STRING), + Column.of("number_of_nodes", INTEGER), + Column.of("number_of_data_nodes", INTEGER), + Column.of("active_primary_shards", INTEGER), + Column.of("active_shards", INTEGER), + Column.of("relocating_shards", INTEGER), + Column.of("initializing_shards", INTEGER), + Column.of("unassigned_shards", INTEGER), + Column.of("timed_out", BOOLEAN))) + .argSpec(ArgSpec.builder().arg("local", Set.of("true", "false")).build()) + .handler(CoreEndpointsProvider::clusterHealth) + .build()); + } + + private static List> clusterHealth(RestEndpointContext ctx) { + NodeClient client = ctx.client(); + if (client == null) { + throw new IllegalStateException( + "the /_cluster/health rest endpoint requires an in-cluster node client"); + } + ClusterHealthRequest request = new ClusterHealthRequest(); + if (Boolean.parseBoolean(ctx.args().get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + Map row = new LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return List.of(row); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java new file mode 100644 index 00000000000..7cce3480cf1 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.EnumMap; +import java.util.Map; +import org.opensearch.sql.spi.rest.RedactionClass; +import org.opensearch.sql.spi.rest.Redactor; + +/** + * The platform-owned map of {@link RedactionClass} to {@link Redactor} that the {@code rest} choke + * point consults when shaping rows. This is deliberately NOT a {@code loadExtensions} SPI: + * redaction is a security control, so an open extension point would let any installed plugin + * register a no-op masker and weaken masking. Instead the registry is populated only by the + * platform through {@link #register} or the constructor. + * + *

OSS leaves it EMPTY, so masking is a pure no-op out of the box. A managed distribution + * supplies per-class {@link Redactor}s through a wrapper patch (precedent: the AOS SQL patch), + * reusing one masker for every endpoint that declares a column of that class. {@link + * RedactionClass#NONE} is a sentinel meaning "never redact" and cannot be registered. + */ +public final class RedactionRegistry { + + private final Map redactors; + + /** An empty registry: every class is a no-op passthrough (the OSS default). */ + public RedactionRegistry() { + this.redactors = new EnumMap<>(RedactionClass.class); + } + + /** A registry pre-populated from a class -> masker map (the managed wrapper seam). */ + public RedactionRegistry(Map redactors) { + this(); + if (redactors != null) { + redactors.forEach(this::register); + } + } + + /** + * Register the masker for one sensitivity class. Rejects {@link RedactionClass#NONE} (the + * never-redact sentinel) and null arguments so the registry stays auditable. + */ + public void register(RedactionClass redactionClass, Redactor redactor) { + if (redactionClass == null || redactionClass == RedactionClass.NONE) { + throw new IllegalArgumentException("cannot register a redactor for redaction class: NONE"); + } + if (redactor == null) { + throw new IllegalArgumentException( + "redactor for redaction class [" + redactionClass + "] must not be null"); + } + redactors.put(redactionClass, redactor); + } + + /** True when no masker is registered (the OSS default): every cell passes through unchanged. */ + public boolean isEmpty() { + return redactors.isEmpty(); + } + + /** + * Mask one cell value for its column's class. A {@code null}/{@code NONE} class, a null value, or + * a class with no registered masker all pass the value through unchanged. + */ + public String mask(RedactionClass redactionClass, String value) { + if (redactionClass == null || redactionClass == RedactionClass.NONE || value == null) { + return value; + } + Redactor redactor = redactors.get(redactionClass); + return redactor == null ? value : redactor.mask(value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java new file mode 100644 index 00000000000..8f9894f1e4e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +/** + * Bridge for sharing the platform {@link RedactionRegistry} between plugin bootstrap (where the SQL + * plugin publishes it) and {@code OpenSearchStorageEngine.getTable} (which applies it at query + * time). Mirrors {@link RestEndpointRegistryHolder}. + * + *

Defaults to an empty registry so the OSS choke point is a pure no-op even before bootstrap + * publishes one, and so {@code getTable} never has to null-check. A managed distribution replaces + * or fills it via a wrapper patch that calls {@link #set} (or {@link RedactionRegistry#register}) + * during {@code createComponents}. + */ +public final class RedactionRegistryHolder { + + private static volatile RedactionRegistry registry = new RedactionRegistry(); + + private RedactionRegistryHolder() {} + + public static void set(RedactionRegistry instance) { + registry = instance == null ? new RedactionRegistry() : instance; + } + + public static RedactionRegistry get() { + return registry; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java index 96779726a7e..d0d0d2dff99 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -17,8 +17,8 @@ /** * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management - * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. - * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + * endpoint resolved against a {@link RestEndpointRegistry} instance, exposing the fixed endpoint + * schema. Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. */ @Getter public class RestCatalogSource implements CatalogSource { @@ -26,19 +26,23 @@ public class RestCatalogSource implements CatalogSource { private final OpenSearchClient client; private final RestSpec spec; private final RestEndpointRegistry.Endpoint endpoint; - private final boolean redact; + private final RedactionRegistry redaction; - public RestCatalogSource(OpenSearchClient client, RestSpec spec) { - this(client, spec, false); + public RestCatalogSource(RestEndpointRegistry registry, RestSpec spec, OpenSearchClient client) { + this(registry, spec, client, new RedactionRegistry()); } - public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { + public RestCatalogSource( + RestEndpointRegistry registry, + RestSpec spec, + OpenSearchClient client, + RedactionRegistry redaction) { this.client = client; this.spec = spec; - this.redact = redact; + this.redaction = redaction; // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); + this.endpoint = registry.resolve(spec.getEndpoint()); + registry.validate(spec); } @Override @@ -48,7 +52,7 @@ public Map getFieldTypes() { @Override public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec, redact); + return new RestRequest(client, endpoint, spec, redaction); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index e64a91ab54a..95da2bcae76 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -20,262 +20,130 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import lombok.Getter; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.ColumnType; +import org.opensearch.sql.spi.rest.RedactionClass; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointHandler; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps - * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so - * the Calcite plan can fix its row type at plan time), and the query args it accepts. + * The read-only endpoint allow-list, built by merging every {@link RestEndpointProvider} (the + * built-in {@link CoreEndpointsProvider} plus any externally contributed providers) into one map of + * endpoint name to an internal {@link Endpoint}. A built-in and an externally contributed endpoint + * are uniform entries here; the built-in provider holds no privileged position. * - *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, - * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. - * Adding an endpoint is a reviewed change here, never arbitrary pass-through. + *

This is the single place the read-only allow-list is enforced. An endpoint that no provider + * registered, including every mutating endpoint, is rejected by {@link #resolve} with a clear + * exception, and an arg the endpoint's {@link ArgSpec} does not accept is rejected by {@link + * #validate}. Adding an endpoint is a reviewed change to a provider, never arbitrary pass-through. */ public final class RestEndpointRegistry { - private RestEndpointRegistry() {} + private final Map registry; - /** Produces the raw rows for an endpoint via a read-only client call. */ - @FunctionalInterface - public interface RowFetcher { - List> fetch(OpenSearchClient client, RestSpec spec); + public RestEndpointRegistry(List providers) { + Map m = new LinkedHashMap<>(); + for (RestEndpointProvider provider : providers) { + for (RestEndpointDefinition definition : provider.getEndpoints()) { + if (m.containsKey(definition.name())) { + throw new IllegalStateException( + "rest endpoint [" + + definition.name() + + "] is registered by more than one provider; endpoint names must be unique"); + } + m.put(definition.name(), new Endpoint(definition)); + } + } + this.registry = m; } - /** A single allow-listed endpoint description. */ + public java.util.Set endpointNames() { + return registry.keySet(); + } + + /** A single allow-listed endpoint, adapted from a provider's {@link RestEndpointDefinition}. */ @Getter public static final class Endpoint { private final String path; private final LinkedHashMap schema; - private final Set allowedArgs; - private final RowFetcher fetcher; - - Endpoint( - String path, - LinkedHashMap schema, - Set allowedArgs, - RowFetcher fetcher) { - this.path = path; - this.schema = schema; - this.allowedArgs = allowedArgs; - this.fetcher = fetcher; - } + // Column name -> sensitivity class, the join key the choke point uses to pick a Redactor. + private final Map redactionClasses; + private final ArgSpec argSpec; + private final RestEndpointHandler handler; - /** Dispatch the read-only call and shape the response into fixed-schema rows. */ - public List toRows(OpenSearchClient client, RestSpec spec) { - return toRows(client, spec, false); + Endpoint(RestEndpointDefinition definition) { + this.path = definition.name(); + this.schema = toExprSchema(definition.schema()); + this.redactionClasses = toRedactionClasses(definition.schema()); + this.argSpec = definition.argSpec(); + this.handler = definition.handler(); } /** - * Shape the response into fixed-schema rows, masking network identifiers when redaction is - * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value - * column is zone-masked. Off by default. + * Invoke the provider's handler and shape the response into fixed-schema rows, masking each + * cell by its column's {@link RedactionClass} through the platform {@link RedactionRegistry}. + * Runs at execution time (scan open). An empty registry (the OSS default) masks nothing. */ - public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { - boolean redactCat = redact && path.startsWith("/_cat"); - boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); + public List toRows(RestEndpointContext ctx, RedactionRegistry redaction) { List out = new ArrayList<>(); - for (Map raw : fetcher.fetch(client, spec)) { + for (Map raw : handler.fetch(ctx)) { LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); - tuple.put( - col.getKey(), - maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); + tuple.put(col.getKey(), maskCell(col.getKey(), col.getValue(), value, redaction)); } out.add(new ExprTupleValue(tuple)); } return out; } - private static ExprValue maskCell( - String column, - ExprType type, - ExprValue value, - boolean redactCat, - boolean redactSettingsValue) { + private ExprValue maskCell( + String column, ExprType type, ExprValue value, RedactionRegistry redaction) { + // Redaction applies only to non-null string cells; a class declared on a non-string column + // (unusual) is ignored so a number is never fed to a text masker. if (type != STRING || value.isNull()) { return value; } - if (redactCat) { - return stringValue(RestResponseRedactor.redact(value.stringValue())); - } - if (redactSettingsValue && "value".equals(column)) { - return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + RedactionClass redactionClass = redactionClasses.getOrDefault(column, RedactionClass.NONE); + if (redactionClass == RedactionClass.NONE) { + return value; } - return value; + return stringValue(redaction.mask(redactionClass, value.stringValue())); } } - private static final Map REGISTRY = buildRegistry(); - - private static Map buildRegistry() { - Map m = new LinkedHashMap<>(); - - // /_cluster/health — single-row cluster health snapshot (read-only monitor action). - LinkedHashMap healthSchema = new LinkedHashMap<>(); - healthSchema.put("cluster_name", STRING); - healthSchema.put("status", STRING); - healthSchema.put("number_of_nodes", INTEGER); - healthSchema.put("number_of_data_nodes", INTEGER); - healthSchema.put("active_primary_shards", INTEGER); - healthSchema.put("active_shards", INTEGER); - healthSchema.put("relocating_shards", INTEGER); - healthSchema.put("initializing_shards", INTEGER); - healthSchema.put("unassigned_shards", INTEGER); - healthSchema.put("timed_out", BOOLEAN); - m.put( - "/_cluster/health", - new Endpoint( - "/_cluster/health", - healthSchema, - Set.of("local"), - (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); - - // /_cat/indices — one row per index (read-only monitor action). - LinkedHashMap catSchema = new LinkedHashMap<>(); - catSchema.put("index", STRING); - catSchema.put("health", STRING); - catSchema.put("pri", INTEGER); - catSchema.put("rep", INTEGER); - catSchema.put("active_shards", INTEGER); - m.put( - "/_cat/indices", - new Endpoint( - "/_cat/indices", - catSchema, - Set.of("health"), - (client, spec) -> client.catIndices(spec.getArgs()))); - - // /_cat/nodes — one row per node with resource state (read-only monitor action). - LinkedHashMap nodesSchema = new LinkedHashMap<>(); - nodesSchema.put("name", STRING); - nodesSchema.put("ip", STRING); - nodesSchema.put("node_role", STRING); - nodesSchema.put("heap_percent", INTEGER); - nodesSchema.put("ram_percent", INTEGER); - nodesSchema.put("cpu", INTEGER); - m.put( - "/_cat/nodes", - new Endpoint( - "/_cat/nodes", - nodesSchema, - Set.of(), - (client, spec) -> client.catNodes(spec.getArgs()))); - - // /_cat/cluster_manager — single row identifying the elected cluster manager node. - LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); - clusterManagerSchema.put("id", STRING); - clusterManagerSchema.put("host", STRING); - clusterManagerSchema.put("ip", STRING); - clusterManagerSchema.put("node", STRING); - m.put( - "/_cat/cluster_manager", - new Endpoint( - "/_cat/cluster_manager", - clusterManagerSchema, - Set.of(), - (client, spec) -> client.catClusterManager(spec.getArgs()))); - - // /_cat/plugins — one row per installed plugin per node (read-only monitor action). - LinkedHashMap pluginsSchema = new LinkedHashMap<>(); - pluginsSchema.put("name", STRING); - pluginsSchema.put("component", STRING); - pluginsSchema.put("version", STRING); - m.put( - "/_cat/plugins", - new Endpoint( - "/_cat/plugins", - pluginsSchema, - Set.of(), - (client, spec) -> client.catPlugins(spec.getArgs()))); - - // /_cat/shards — one row per shard (read-only monitor action). - LinkedHashMap shardsSchema = new LinkedHashMap<>(); - shardsSchema.put("index", STRING); - shardsSchema.put("shard", INTEGER); - shardsSchema.put("prirep", STRING); - shardsSchema.put("state", STRING); - shardsSchema.put("node", STRING); - m.put( - "/_cat/shards", - new Endpoint( - "/_cat/shards", - shardsSchema, - Set.of(), - (client, spec) -> client.catShards(spec.getArgs()))); - - // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). - LinkedHashMap stateSchema = new LinkedHashMap<>(); - stateSchema.put("cluster_name", STRING); - stateSchema.put("state_uuid", STRING); - stateSchema.put("version", LONG); - stateSchema.put("cluster_manager_node", STRING); - m.put( - "/_cluster/state", - new Endpoint( - "/_cluster/state", - stateSchema, - Set.of(), - (client, spec) -> List.of(client.clusterState(spec.getArgs())))); - - // /_cluster/settings — one row per configured setting (persistent/transient tier). - LinkedHashMap settingsSchema = new LinkedHashMap<>(); - settingsSchema.put("setting", STRING); - settingsSchema.put("value", STRING); - settingsSchema.put("tier", STRING); - m.put( - "/_cluster/settings", - new Endpoint( - "/_cluster/settings", - settingsSchema, - Set.of(), - (client, spec) -> client.clusterSettings(spec.getArgs()))); - - // /_resolve/index — one row per resolved index/alias/data_stream name. - LinkedHashMap resolveSchema = new LinkedHashMap<>(); - resolveSchema.put("name", STRING); - resolveSchema.put("type", STRING); - m.put( - "/_resolve/index", - new Endpoint( - "/_resolve/index", - resolveSchema, - Set.of("expand_wildcards"), - (client, spec) -> client.resolveIndex(spec.getArgs()))); - - return m; - } - /** - * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * Resolve an allow-listed endpoint. Anything no provider registered (unknown path, mutating verb, * {@code /services/*}, plugin admin endpoints) is refused here. */ - public static Endpoint resolve(String path) { + public Endpoint resolve(String path) { if (path == null || path.isBlank()) { throw new IllegalArgumentException( "rest endpoint must be a non-empty path. Supported read-only endpoints: " - + REGISTRY.keySet()); + + registry.keySet()); } - Endpoint endpoint = REGISTRY.get(path); + Endpoint endpoint = registry.get(path); if (endpoint == null) { throw new IllegalArgumentException( "rest endpoint [" + path + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " - + REGISTRY.keySet()); + + registry.keySet()); } return endpoint; } - /** Validate that every supplied query arg is accepted by the endpoint. */ - public static void validate(RestSpec spec) { + /** Validate the count, the reserved timeout token, and every supplied query arg. */ + public void validate(RestSpec spec) { Endpoint endpoint = resolve(spec.getEndpoint()); if (spec.getCount() != null && spec.getCount() < 0) { throw new IllegalArgumentException( @@ -294,69 +162,46 @@ public static void validate(RestSpec spec) { "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); } if (spec.getArgs() != null) { + ArgSpec argSpec = endpoint.getArgSpec(); for (String arg : spec.getArgs().keySet()) { - if (!endpoint.getAllowedArgs().contains(arg)) { + if (!argSpec.allows(arg)) { throw new IllegalArgumentException( "rest endpoint [" + spec.getEndpoint() + "] does not accept arg [" + arg + "]. Allowed args: " - + endpoint.getAllowedArgs()); + + argSpec.allowedArgs()); } - validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + argSpec.validateValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); } } } - // Allowed value domains for the get-args that are applied server-side. Keys are validated against - // the per-endpoint allow-list above; values are validated here so a user-supplied value is never - // passed unchecked into an admin transport request. - private static final Map> ARG_VALUE_DOMAINS = - Map.of( - "local", Set.of("true", "false"), - "health", Set.of("green", "yellow", "red")); - - private static final Set EXPAND_WILDCARDS_VALUES = - Set.of("open", "closed", "hidden", "none", "all"); + private static LinkedHashMap toExprSchema(List columns) { + LinkedHashMap schema = new LinkedHashMap<>(); + for (Column column : columns) { + schema.put(column.name(), toExprType(column.type())); + } + return schema; + } - /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ - private static void validateArgValue(String endpoint, String arg, String value) { - Set domain = ARG_VALUE_DOMAINS.get(arg); - if (domain != null) { - if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [" - + arg - + "] has an unsupported value [" - + value - + "]. Allowed values: " - + domain); - } - } else if ("expand_wildcards".equals(arg)) { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { - if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - } + private static Map toRedactionClasses(List columns) { + LinkedHashMap classes = new LinkedHashMap<>(); + for (Column column : columns) { + classes.put(column.name(), column.redaction()); } + return classes; + } + + private static ExprType toExprType(ColumnType type) { + return switch (type) { + case STRING -> STRING; + case INTEGER -> INTEGER; + case LONG -> LONG; + case DOUBLE -> DOUBLE; + case BOOLEAN -> BOOLEAN; + }; } private static ExprValue coerce(String column, ExprType type, Object value) { @@ -393,7 +238,6 @@ private static ExprValue coerce(String column, ExprType type, Object value) { return stringValue(String.valueOf(value)); } - /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ private static Number toNumber(Object value) { if (value instanceof Number n) { return n; @@ -408,7 +252,6 @@ private static Number toNumber(Object value) { return Long.parseLong(s); } - /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ private static boolean toBoolean(Object value) { if (value instanceof Boolean b) { return b; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java new file mode 100644 index 00000000000..18669c05187 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +/** + * Bridge for sharing the merged {@link RestEndpointRegistry} between plugin bootstrap (where the + * SQL plugin builds it from the built-in provider plus every provider discovered via {@code + * ExtensiblePlugin.loadExtensions}) and {@code OpenSearchStorageEngine.getTable} (which resolves a + * {@code rest} endpoint at query time). + * + *

Why a static holder: {@code loadExtensions} runs during node bootstrap, before the Node-level + * Guice injector exists, so the merged registry cannot be injected into the storage engine. + * Publishing it here once at bootstrap lets the storage engine read the same instance without going + * through the injector. Mirrors {@code AnalyticsExecutorHolder}. + */ +public final class RestEndpointRegistryHolder { + + private static volatile RestEndpointRegistry registry; + + private RestEndpointRegistryHolder() {} + + public static void set(RestEndpointRegistry instance) { + registry = instance; + } + + public static RestEndpointRegistry get() { + return registry; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index 868dabdd64e..8cfb8f697e3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -9,41 +9,49 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; +import org.opensearch.transport.client.node.NodeClient; /** - * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * Dispatches an allow-listed, read-only management endpoint through the endpoint's handler under * the caller's security thread-context and returns the response shaped to the endpoint's fixed * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to - * the system-index scan family. + * the system-index scan family. This is the lazy scan: the handler runs here at {@link #search} + * (execution), never at planning time. */ public class RestRequest implements OpenSearchSystemRequest { private final OpenSearchClient client; private final RestEndpointRegistry.Endpoint endpoint; private final RestSpec spec; - private final boolean redact; + private final RedactionRegistry redaction; public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { - this(client, endpoint, spec, false); + this(client, endpoint, spec, new RedactionRegistry()); } public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec, - boolean redact) { + RedactionRegistry redaction) { this.client = client; this.endpoint = endpoint; this.spec = spec; - this.redact = redact; + this.redaction = redaction; } @Override public List search() { - List rows = endpoint.toRows(client, spec, redact); + // The context's transport client is only used by externally contributed providers; the + // built-in provider's handlers hold their own client. Sourced from the same OpenSearchClient + // the storage engine uses, so it runs under the caller's security thread-context. + NodeClient nodeClient = client == null ? null : client.getNodeClient().orElse(null); + RestEndpointContext ctx = RestEndpointContext.of(spec.getArgs(), nodeClient); + List rows = endpoint.toRows(ctx, redaction); if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { return rows.subList(0, spec.getCount()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java deleted file mode 100644 index fd674dccde4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import java.util.regex.Pattern; - -/** - * Masks network identifiers in rest command cell values. Enabled per deployment via {@code - * plugins.ppl.rest.redaction.enabled}; off by default. - */ -public final class RestResponseRedactor { - - private RestResponseRedactor() {} - - private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; - private static final Pattern IPV4 = - Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); - private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); - private static final Pattern EC2_HOST = - Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); - private static final Pattern IPV6 = - Pattern.compile( - "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" - + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", - Pattern.CASE_INSENSITIVE); - private static final Pattern AZ_NAME = - Pattern.compile( - "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", - Pattern.CASE_INSENSITIVE); - - private record Mask(Pattern pattern, String replacement) {} - - private static final List MASKS = - List.of( - new Mask(IPV4, "x.x.x.x"), - new Mask(INET, "inet[/x.x.x.x:y]"), - new Mask(EC2_HOST, ""), - new Mask(IPV6, "x.x.x.x"), - new Mask(AZ_NAME, "xx-xxxxx-xx")); - - /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ - public static String redact(String text) { - if (text == null || text.isEmpty()) { - return text; - } - String out = text; - for (Mask mask : MASKS) { - out = mask.pattern().matcher(out).replaceAll(mask.replacement()); - } - return out; - } - - /** Mask availability-zone names only. */ - public static String maskAvailabilityZone(String text) { - if (text == null || text.isEmpty()) { - return text; - } - return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java deleted file mode 100644 index d425c361e6a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import org.opensearch.common.settings.SettingsFilter; - -/** - * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' - * /_cluster/settings'} fetcher. - * - *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with - * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the - * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads - * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the - * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction - * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. - * - *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in - * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link - * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code - * getRestHandlers} fires lets the fetcher read the same instance without going back through the - * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. - */ -public final class RestSettingsFilterHolder { - - private static volatile SettingsFilter settingsFilter; - - private RestSettingsFilterHolder() {} - - public static void set(SettingsFilter instance) { - settingsFilter = instance; - } - - public static SettingsFilter get() { - return settingsFilter; - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java deleted file mode 100644 index 33be6cc8482..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.client; - -import static java.util.stream.Collectors.toSet; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.opensearch.action.admin.cluster.state.ClusterStateRequest; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; -import org.opensearch.transport.client.node.NodeClient; - -/** - * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using - * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. - */ -class OpenSearchNodeClientClusterSettingsFilterTest { - - @AfterEach - void clearHolder() { - RestSettingsFilterHolder.set(null); - } - - private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { - NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); - ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); - when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) - .thenReturn(stateResp); - when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); - when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); - return new OpenSearchNodeClient(nodeClient); - } - - @Test - void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("plugins.secret.token", "supersecret") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Publish a filter that redacts the secret key, exactly as the native endpoint would. - RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); - - List> rows = client.clusterSettings(Map.of()); - Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); - assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); - } - - @Test - void clusterSettingsRedactsByGlobPattern() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("s3.client.default.secret_key", "AKIAEXAMPLE") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); - - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable")); - assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); - } - - @Test - void clusterSettingsFailsClosedWhenNoFilterPublished() { - Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw - // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter - // before any query, so this path is unreachable in cluster. - assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 0c570098924..4d77df2c992 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -17,7 +17,6 @@ import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -79,10 +78,8 @@ void pluginNonDynamicSettings() { @Test void restSettingsAreNonDynamic() { - assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); - assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index 102ec4da8f7..cae3858a83d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -16,6 +16,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -24,6 +25,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.SystemIndexUtils; @@ -35,6 +39,13 @@ class OpenSearchStorageEngineTest { @Mock private Settings settings; + @BeforeEach + void publishRestRegistry() { + // restTable() reads the merged registry from the holder (published by SQLPlugin in production); + // publish a built-in-only registry here so the rest endpoints resolve in this unit test. + RestEndpointRegistryHolder.set(new RestEndpointRegistry(List.of(new CoreEndpointsProvider()))); + } + @Test public void getTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); @@ -65,11 +76,10 @@ public void getSystemTable() { public void getRestTableAllowedByWildcard() { when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) .thenReturn(List.of("*")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); assertTrue(table instanceof OpenSearchCatalogTable); @@ -78,12 +88,11 @@ public void getRestTableAllowedByWildcard() { @Test public void getRestTableAllowedBySubset() { when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + .thenReturn(List.of("/_cluster/health")); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); assertTrue( engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) instanceof OpenSearchCatalogTable); @@ -91,12 +100,13 @@ public void getRestTableAllowedBySubset() { @Test public void getRestTableRejectedWhenEndpointNotInSubset() { + // The endpoint resolves in the registry but is absent from the (non-empty) enabled subset. when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); + .thenReturn(List.of("/_some/other")); OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); IllegalArgumentException e = assertThrows( IllegalArgumentException.class, @@ -112,7 +122,7 @@ public void getRestTableDisabledWhenListEmpty() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); String name = SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); IllegalArgumentException e = assertThrows( IllegalArgumentException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java new file mode 100644 index 00000000000..2ba6b6707e9 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Proves the built-in {@code /_cluster/health} provider fetches through the transport node client + * the context carries (the same seam an external provider uses) and flattens the response to the + * fixed schema, holding no reference to the sql storage client. + */ +class CoreEndpointsProviderTest { + + @Test + void clusterHealthFetchesViaContextNodeClient() { + ClusterHealthResponse response = mock(ClusterHealthResponse.class); + when(response.getClusterName()).thenReturn("test-cluster"); + when(response.getStatus()).thenReturn(ClusterHealthStatus.GREEN); + when(response.getNumberOfNodes()).thenReturn(3); + when(response.getNumberOfDataNodes()).thenReturn(2); + when(response.getActivePrimaryShards()).thenReturn(5); + when(response.getActiveShards()).thenReturn(10); + when(response.getRelocatingShards()).thenReturn(0); + when(response.getInitializingShards()).thenReturn(1); + when(response.getUnassignedShards()).thenReturn(4); + when(response.isTimedOut()).thenReturn(false); + + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().health(any(ClusterHealthRequest.class)).actionGet()) + .thenReturn(response); + + RestEndpointRegistry registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + List rows = + registry + .resolve("/_cluster/health") + .toRows(RestEndpointContext.of(Map.of(), nodeClient), new RedactionRegistry()); + + assertEquals(1, rows.size()); + Map row = rows.get(0).tupleValue(); + assertEquals("test-cluster", row.get("cluster_name").stringValue()); + assertEquals("green", row.get("status").stringValue()); + assertEquals(3, row.get("number_of_nodes").integerValue()); + assertFalse(row.get("timed_out").booleanValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java new file mode 100644 index 00000000000..3c6f7766d47 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.spi.rest.ColumnType.STRING; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.RedactionClass; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; + +/** + * Proves the core property of class-based redaction: a sensitivity class is declared ONCE on a + * column and one platform {@link org.opensearch.sql.spi.rest.Redactor} registered for that class is + * reused by EVERY endpoint that has such a column, endpoint- and column-name-agnostic. Two + * unrelated fake endpoints, each with an {@link RedactionClass#IP} column of a different name, are + * both masked by the single registered IP redactor; a column of another class is untouched; and an + * empty registry (the OSS default) masks nothing. + */ +class RedactionByClassTest { + + private static final String MASK = "x.x.x.x"; + + /** + * Two unrelated endpoints, each with an IP-classed column (different names) plus a NONE column. + */ + private static final class TwoIpEndpointsProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_fake/alpha") + .schema( + List.of( + Column.of("addr", STRING, RedactionClass.IP), + Column.of("label", STRING, RedactionClass.NONE))) + .argSpec(ArgSpec.NONE) + .handler(ctx -> List.of(Map.of("addr", "10.0.0.7", "label", "10.0.0.7"))) + .build(), + RestEndpointDefinition.builder() + .name("/_fake/beta") + .schema( + List.of( + Column.of("host_ip", STRING, RedactionClass.IP), Column.of("note", STRING))) + .argSpec(ArgSpec.NONE) + .handler(ctx -> List.of(Map.of("host_ip", "192.168.1.1", "note", "192.168.1.1"))) + .build()); + } + } + + private static RestEndpointRegistry registry() { + return new RestEndpointRegistry(List.of(new TwoIpEndpointsProvider())); + } + + private static RestEndpointContext ctx() { + return RestEndpointContext.of(Map.of(), null); + } + + @Test + void oneRedactorMasksTheIpColumnOfEveryEndpoint() { + // Declare the IP masker ONCE; it must apply to both endpoints' IP columns regardless of name. + RedactionRegistry redaction = new RedactionRegistry(); + redaction.register(RedactionClass.IP, value -> MASK); + + RestEndpointRegistry registry = registry(); + + Map alpha = + registry.resolve("/_fake/alpha").toRows(ctx(), redaction).get(0).tupleValue(); + assertEquals(MASK, alpha.get("addr").stringValue(), "alpha IP column masked"); + // A NONE-class column is never masked, even though its value is an address. + assertEquals("10.0.0.7", alpha.get("label").stringValue(), "NONE-class column untouched"); + + Map beta = + registry.resolve("/_fake/beta").toRows(ctx(), redaction).get(0).tupleValue(); + assertEquals( + MASK, beta.get("host_ip").stringValue(), "beta IP column masked by the same redactor"); + assertEquals("192.168.1.1", beta.get("note").stringValue(), "default NONE column untouched"); + } + + @Test + void emptyRegistryIsANoOp() { + RedactionRegistry empty = new RedactionRegistry(); + RestEndpointRegistry registry = registry(); + + Map alpha = + registry.resolve("/_fake/alpha").toRows(ctx(), empty).get(0).tupleValue(); + // No redactor registered for IP: the value passes through unchanged. + assertEquals("10.0.0.7", alpha.get("addr").stringValue()); + assertEquals("10.0.0.7", alpha.get("label").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java index 8676373d2a6..b8111fadadc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -18,6 +17,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -25,25 +26,53 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.ColumnType; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, - * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) - * path. + * Covers the {@code rest} {@link RestCatalogSource} against the PR1 endpoint set (only {@code + * /_cluster/health}): fixed endpoint schema, allow-list enforcement, response row shaping and + * truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) path. Schema and gating + * resolve against a registry built from the built-in {@link CoreEndpointsProvider}; row shaping and + * truncation use a fake provider returning canned rows, independent of the health transport fetch. */ @ExtendWith(MockitoExtension.class) class RestCatalogSourceTest { @Mock private OpenSearchClient client; + private RestEndpointRegistry registry; + + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + } + private RestSpec healthSpec() { return new RestSpec("/_cluster/health", Map.of(), null, null); } + private static RestEndpointRegistry fakeHealthRegistry(List> rows) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .schema( + List.of( + Column.of("status", ColumnType.STRING), + Column.of("number_of_nodes", ColumnType.INTEGER))) + .handler(ctx -> rows) + .build()); + return new RestEndpointRegistry(List.of(provider)); + } + @Test void getFieldTypesReturnsFixedEndpointSchema() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); Map fieldTypes = source.getFieldTypes(); assertThat(fieldTypes, hasEntry("status", STRING)); assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); @@ -51,12 +80,12 @@ void getFieldTypesReturnsFixedEndpointSchema() { @Test void isScannable() { - assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); + assertTrue(new RestCatalogSource(registry, healthSpec(), client).isScannable()); } @Test void implementV2IsUnsupported() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); } @@ -65,7 +94,8 @@ void constructorRejectsNonAllowListedEndpoint() { assertThrows( IllegalArgumentException.class, () -> - new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + new RestCatalogSource( + registry, new RestSpec("/_cluster/reroute", Map.of(), null, null), client)); } @Test @@ -74,14 +104,18 @@ void constructorRejectsDisallowedArg() { IllegalArgumentException.class, () -> new RestCatalogSource( - client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + registry, + new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null), + client)); } @Test void constructorRejectsNegativeCount() { assertThrows( IllegalArgumentException.class, - () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); + () -> + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), -1, null), client)); } @Test @@ -89,17 +123,18 @@ void constructorRejectsTimeoutArg() { assertThrows( IllegalArgumentException.class, () -> - new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), null, "5s"), client)); } @Test void restRequestShapesResponseRows() { + when(client.getNodeClient()).thenReturn(Optional.empty()); Map health = new LinkedHashMap<>(); health.put("status", "green"); health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + RestCatalogSource source = + new RestCatalogSource(fakeHealthRegistry(List.of(health)), healthSpec(), client); List rows = source.createRequest().search(); assertEquals(1, rows.size()); assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); @@ -108,16 +143,15 @@ void restRequestShapesResponseRows() { @Test void countTruncatesRows() { - Map idx1 = new LinkedHashMap<>(); - idx1.put("index", "a"); - Map idx2 = new LinkedHashMap<>(); - idx2.put("index", "b"); - when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - + // count=0 exercises the truncation path (subList to empty) over a single-row response. + when(client.getNodeClient()).thenReturn(Optional.empty()); + Map health = new LinkedHashMap<>(); + health.put("status", "green"); RestCatalogSource source = - new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + new RestCatalogSource( + fakeHealthRegistry(List.of(health)), + new RestSpec("/_cluster/health", Map.of(), 0, null), + client); + assertTrue(source.createRequest().search().isEmpty()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java new file mode 100644 index 00000000000..00194ad9b0f --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.spi.rest.ColumnType.STRING; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Proves the {@code rest} framework treats the built-in {@link CoreEndpointsProvider} and an + * externally contributed {@link RestEndpointProvider} as uniform clients of one registry: endpoints + * from BOTH resolve, validate against the same allow-list, and fetch rows the same way. The + * built-in provider holds no privileged position. + */ +class RestEndpointExtensibilityTest { + + /** A stand-in external plugin provider: contributes one endpoint that echoes a query arg. */ + private static final class FakeEchoProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_plugin/echo") + .schema(List.of(Column.of("message", STRING))) + .argSpec(ArgSpec.builder().arg("text").build()) + .handler( + ctx -> List.of(Map.of("message", ctx.args().getOrDefault("text", "default")))) + .build()); + } + } + + private RestEndpointRegistry mergedRegistry() { + return new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), new FakeEchoProvider())); + } + + @Test + void bothBuiltInAndExternalEndpointsResolve() { + RestEndpointRegistry registry = mergedRegistry(); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertEquals("/_plugin/echo", registry.resolve("/_plugin/echo").getPath()); + + assertTrue(registry.endpointNames().contains("/_cluster/health")); + assertTrue(registry.endpointNames().contains("/_plugin/echo")); + } + + @Test + void externalEndpointFetchesThroughTheSamePath() { + RestEndpointRegistry.Endpoint echo = mergedRegistry().resolve("/_plugin/echo"); + List rows = + echo.toRows(RestEndpointContext.of(Map.of("text", "hi"), null), new RedactionRegistry()); + assertEquals(1, rows.size()); + assertEquals("hi", rows.get(0).tupleValue().get("message").stringValue()); + } + + @Test + void externalEndpointArgsValidatedByTheSameAllowList() { + RestEndpointRegistry registry = mergedRegistry(); + // Declared arg is accepted. + registry.validate(new RestSpec("/_plugin/echo", Map.of("text", "x"), null, null)); + // Undeclared arg is rejected by the same validation path that guards built-in endpoints. + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> + registry.validate( + new RestSpec("/_plugin/echo", Map.of("not_allowed", "x"), null, null))); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void duplicateEndpointNameAcrossProvidersIsRejected() { + RestEndpointProvider shadowsCore = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .schema(List.of(Column.of("x", STRING))) + .handler(ctx -> List.of()) + .build()); + IllegalStateException ex = + assertThrows( + IllegalStateException.class, + () -> new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadowsCore))); + assertTrue(ex.getMessage().contains("more than one provider")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index c5a978bb495..179bc2f2747 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -8,131 +8,91 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.ColumnType; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; -@ExtendWith(MockitoExtension.class) +/** + * Covers the {@code rest} {@link RestEndpointRegistry} against the PR1 endpoint set (only {@code + * /_cluster/health}): allow-list resolution, arg validation, count/timeout gating, value coercion, + * and fixed-schema row shaping. Coercion and shaping are exercised through a fake provider that + * returns canned rows, so they stay independent of how any one endpoint fetches. Redaction-by-class + * is covered separately in {@link RedactionByClassTest}; here rows are shaped with an empty {@link + * RedactionRegistry} (no-op). + */ class RestEndpointRegistryTest { - @Mock private OpenSearchClient client; + private RestEndpointRegistry registry; - @Test - void resolveAllowListedEndpoint() { - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - assertEquals("/_cluster/health", endpoint.getPath()); - assertEquals(STRING, endpoint.getSchema().get("status")); - assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); } - @Test - void resolveRejectsNonAllowListedEndpoint() { - // A mutating endpoint is simply absent from the registry and is refused here. - assertThrows( - IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); - assertThrows( - IllegalArgumentException.class, - () -> RestEndpointRegistry.resolve("/services/server/info")); + private static RestEndpointContext ctx(Map args) { + return RestEndpointContext.of(args, null); } - @Test - void validateRejectsUnknownArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + private static RestEndpointRegistry.Endpoint fakeEndpoint( + List schema, Map row) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_test/probe") + .schema(schema) + .handler(context -> List.of(row)) + .build()); + return new RestEndpointRegistry(List.of(provider)).resolve("/_test/probe"); } @Test - void validateAcceptsAllowedArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); - RestEndpointRegistry.validate(spec); // no throw + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = registry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); } @Test - void catEndpointsRedactAddressesWhenRedactionEnabled() { - Map node = new LinkedHashMap<>(); - node.put("name", "ip-10-0-0-7"); - node.put("ip", "10.0.0.7"); - node.put("node_role", "dir"); - node.put("heap_percent", 44); - node.put("ram_percent", 95); - node.put("cpu", 2); - when(client.catNodes(any())).thenReturn(List.of(node)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); - RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("", redacted.get("name").stringValue()); - assertEquals(44, redacted.get("heap_percent").integerValue()); - - Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); - assertEquals("10.0.0.7", plain.get("ip").stringValue()); - assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint, and any endpoint deferred out of PR1, is simply absent and refused here. + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cluster/reroute")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cat/nodes")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/services/server/info")); } @Test - void catClusterManagerRedactsHostAndIp() { - Map row = new LinkedHashMap<>(); - row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); - row.put("host", "10.0.0.7"); - row.put("ip", "10.0.0.7"); - row.put("node", "71d03b567bb755839a73d437b2b066d4"); - when(client.catClusterManager(any())).thenReturn(List.of(row)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); - RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("host").stringValue()); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); - assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> registry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(null)); } @Test - void nonCatEndpointNotRedactedEvenWhenEnabled() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "10.0.0.7"); - health.put("status", "green"); - health.put("number_of_nodes", 3); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); } @Test - void clusterSettingsMasksAvailabilityZoneInValue() { - Map setting = new LinkedHashMap<>(); - setting.put("setting", "cluster.routing.allocation.awareness.attributes"); - setting.put("value", "zone:us-east-1a"); - setting.put("tier", "persistent"); - when(client.clusterSettings(any())).thenReturn(List.of(setting)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); - RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); - assertEquals( - "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); - assertEquals("persistent", row.get("tier").stringValue()); + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + registry.validate(spec); // no throw } @Test @@ -140,96 +100,53 @@ void validateRejectsDroppedLevelArg() { // level was dropped (no-op against the fixed cluster-level health schema); now unknown. RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("does not accept arg")); } - @Test - void validateRejectsDroppedFlatSettingsArg() { - // flat_settings was dropped (redundant: settings are already flattened to dotted keys). - RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsValidArgValues() { - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); - } - @Test void validateRejectsBadArgValue() { - IllegalArgumentException health = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); - assertTrue(health.getMessage().contains("unsupported value")); - IllegalArgumentException local = assertThrows( IllegalArgumentException.class, () -> - RestEndpointRegistry.validate( + registry.validate( new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); assertTrue(local.getMessage().contains("unsupported value")); - - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec( - "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); - } - - @Test - void resolveRejectsBlankEndpoint() { - IllegalArgumentException emptyEx = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); - assertTrue(emptyEx.getMessage().contains("non-empty path")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); } @Test void validateRejectsNegativeCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), -1, null); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("non-negative")); } @Test void validateAcceptsZeroCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); - RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), 0, null); + registry.validate(spec); // no throw: 0 is a valid limit } @Test void validateRejectsTimeoutArg() { RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); assertTrue(ex.getMessage().contains("timeout")); } @Test void coerceParsesNumericStringValues() { - // The cat JSON API returns numeric columns as strings; coerce must parse them. - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "3"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - + // The transport/JSON layer may return numeric columns as strings; coerce must parse them. + RestEndpointRegistry.Endpoint endpoint = + fakeEndpoint( + List.of( + Column.of("status", ColumnType.STRING), + Column.of("number_of_nodes", ColumnType.INTEGER)), + Map.of("status", "green", "number_of_nodes", "3")); + List rows = endpoint.toRows(ctx(Map.of()), new RedactionRegistry()); assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); } @@ -237,54 +154,39 @@ void coerceParsesNumericStringValues() { void coerceThrowsClearErrorOnUncoercibleValue() { // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), // not a raw ClassCastException / NumberFormatException (HTTP 500). - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "not-a-number"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestEndpointRegistry.Endpoint endpoint = + fakeEndpoint( + List.of( + Column.of("status", ColumnType.STRING), + Column.of("number_of_nodes", ColumnType.INTEGER)), + Map.of("status", "green", "number_of_nodes", "not-a-number")); IllegalArgumentException ex = assertThrows( IllegalArgumentException.class, - () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + () -> endpoint.toRows(ctx(Map.of()), new RedactionRegistry())); assertTrue(ex.getMessage().contains("number_of_nodes")); assertTrue(ex.getMessage().contains("not-a-number")); } @Test - void clusterHealthRowsAreShapedToFixedSchema() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "test-cluster"); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - + void rowsAreShapedToFixedSchema() { + Map raw = new LinkedHashMap<>(); + raw.put("cluster_name", "test-cluster"); + raw.put("status", "green"); + raw.put("number_of_nodes", 1); + RestEndpointRegistry.Endpoint endpoint = + fakeEndpoint( + List.of( + Column.of("cluster_name", ColumnType.STRING), + Column.of("status", ColumnType.STRING), + Column.of("number_of_nodes", ColumnType.INTEGER), + Column.of("relocating_shards", ColumnType.INTEGER)), + raw); + List rows = endpoint.toRows(ctx(Map.of()), new RedactionRegistry()); assertEquals(1, rows.size()); assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - // a declared column the action did not return becomes null, never absent. + // a declared column the handler did not return becomes null, never absent. assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); } - - @Test - void catIndicesRowsAreShapedToFixedSchema() { - Map idx = new LinkedHashMap<>(); - idx.put("index", "books"); - idx.put("health", "yellow"); - idx.put("pri", 1); - idx.put("rep", 1); - when(client.catIndices(any())).thenReturn(List.of(idx)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); - List rows = - endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); - - assertEquals(1, rows.size()); - assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); - assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); - } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java deleted file mode 100644 index 3b038306da5..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class RestResponseRedactorTest { - - @Test - void masksIpv4() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); - assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); - assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); - } - - @Test - void doesNotMaskInvalidOrPartialIpv4() { - assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); - assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); - assertEquals("44", RestResponseRedactor.redact("44")); - } - - @Test - void masksEc2HostName() { - assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); - assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); - assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); - assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); - } - - @Test - void masksFullIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); - } - - @Test - void masksCompressedIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); - } - - @Test - void masksInetAddress() { - assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); - } - - @Test - void masksAvailabilityZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); - // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); - assertEquals( - "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); - } - - @Test - void maskAvailabilityZoneMasksOnlyZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); - assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); - assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); - } - - @Test - void leavesNonAddressesIntact() { - assertEquals( - "e4e136ea81e27370ff73cf753ba22d39", - RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); - assertEquals( - "data,ingest,remote_cluster_client", - RestResponseRedactor.redact("data,ingest,remote_cluster_client")); - assertEquals( - "x.x.x.x 44 95 imr - e4e136ea", - RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); - } - - @Test - void handlesNullAndEmpty() { - assertEquals(null, RestResponseRedactor.redact(null)); - assertEquals("", RestResponseRedactor.redact("")); - assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); - assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); - } -} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 48ad5c0b0a8..cbeee5f2e00 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -104,6 +104,11 @@ import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RedactionRegistry; +import org.opensearch.sql.opensearch.storage.rest.RedactionRegistryHolder; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; @@ -134,6 +139,7 @@ import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.CreateAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.threadpool.ExecutorBuilder; @@ -153,6 +159,7 @@ public class SQLPlugin extends Plugin private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class); private List executionEngineExtensions = List.of(); + private List restEndpointProviders = List.of(); private ClusterService clusterService; /** Settings should be inited when bootstrap the plugin. */ @@ -181,6 +188,17 @@ public void loadExtensions(ExtensionLoader loader) { executionEngineExtensions.size(), executionEngineExtensions.stream().map(e -> e.getClass().getSimpleName()).toList()); } + + List restProviders = loader.loadExtensions(RestEndpointProvider.class); + this.restEndpointProviders = restProviders != null ? List.copyOf(restProviders) : List.of(); + } + + private void publishRestCommandRegistries() { + List providers = new ArrayList<>(); + providers.add(new CoreEndpointsProvider()); + providers.addAll(this.restEndpointProviders); + RestEndpointRegistryHolder.set(new RestEndpointRegistry(providers)); + RedactionRegistryHolder.set(new RedactionRegistry()); } @Override @@ -197,10 +215,6 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); - // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can - // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); - return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), @@ -368,6 +382,9 @@ public Collection createComponents( this.clusterService = clusterService; this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; + + publishRestCommandRegistries(); + this.dataSourceService = createDataSourceService(); dataSourceService.createDataSource(defaultOpenSearchDataSourceMetadata()); LocalClusterState.state().setClusterService(clusterService); diff --git a/ppl-rest-spi/build.gradle b/ppl-rest-spi/build.gradle new file mode 100644 index 00000000000..ce91f160f7a --- /dev/null +++ b/ppl-rest-spi/build.gradle @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * The `rest` command extension SPI. + * + * This module is intentionally thin: it declares ONLY the interfaces a plugin implements to + * contribute read-only `rest` endpoints, plus the JDK-typed row/column/arg value objects those + * interfaces exchange. It depends on the OpenSearch core artifact ONLY (for the transport client + * handed to a handler) and on NO sql module, so external plugins can compileOnly it without a + * dependency cycle and without a cross-classloader type-identity problem (row values are plain + * java.lang types). + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'com.diffplug.spotless' +} + +dependencies { + // Core only. Deliberately no `project(':core')` / `project(':opensearch')` dependency so the + // sql plugin (which depends on this module) never forms a cycle, and so an external plugin can + // depend on this module alone. + compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + + testImplementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.3') + testImplementation('org.junit.jupiter:junit-jupiter-params:5.9.3') + testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine') + testRuntimeOnly('org.junit.platform:junit-platform-launcher') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +spotless { + java { + target fileTree('.') { + include '**/*.java' + exclude '**/build/**', '**/build-*/**' + } + importOrder() + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java new file mode 100644 index 00000000000..6e88417792e --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java @@ -0,0 +1,111 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * The query args a {@code rest} endpoint accepts, and the allowed value domain of each. This is the + * per-endpoint half of the allow-list: an arg the endpoint does not declare is rejected, and a + * declared arg whose value falls outside its domain is rejected. Both checks are enforced uniformly + * by the sql registry, so a provider expresses its policy as data rather than as code. + * + *

An arg declared with an empty domain accepts any value. An arg declared as a {@code csvArg} + * validates each comma-separated token against the domain (e.g. {@code expand_wildcards}). + */ +public final class ArgSpec { + + public static final ArgSpec NONE = builder().build(); + + // arg name -> allowed values (empty set means "any value allowed"). + private final Map> valueDomains; + // subset of arg names whose value is a comma-separated list validated token-by-token. + private final Set csvArgs; + + private ArgSpec(Map> valueDomains, Set csvArgs) { + this.valueDomains = valueDomains; + this.csvArgs = csvArgs; + } + + public Set allowedArgs() { + return valueDomains.keySet(); + } + + public boolean allows(String arg) { + return valueDomains.containsKey(arg); + } + + /** + * Validate the value of an accepted arg against its domain. No-op if the arg has no domain (any + * value allowed). Throws {@link IllegalArgumentException} with a clear client error otherwise. + */ + public void validateValue(String endpoint, String arg, String value) { + Set domain = valueDomains.get(arg); + if (domain == null || domain.isEmpty()) { + return; + } + if (csvArgs.contains(arg)) { + if (value == null || value.isBlank()) { + throw unsupported(endpoint, arg, value, domain); + } + for (String token : value.toLowerCase(Locale.ROOT).split(",")) { + if (!domain.contains(token.trim())) { + throw unsupported(endpoint, arg, value, domain); + } + } + } else if (value == null || !domain.contains(value.toLowerCase(Locale.ROOT))) { + throw unsupported(endpoint, arg, value, domain); + } + } + + private static IllegalArgumentException unsupported( + String endpoint, String arg, String value, Set domain) { + return new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link ArgSpec}. Declaration order is preserved for stable error messages. */ + public static final class Builder { + private final LinkedHashMap> valueDomains = new LinkedHashMap<>(); + private final Set csvArgs = new LinkedHashSet<>(); + + public Builder arg(String name) { + valueDomains.put(name, Set.of()); + return this; + } + + public Builder arg(String name, Set domain) { + valueDomains.put(name, Set.copyOf(domain)); + return this; + } + + /** Accept {@code name} as a comma-separated list; every token must be in {@code domain}. */ + public Builder csvArg(String name, Set domain) { + valueDomains.put(name, Set.copyOf(domain)); + csvArgs.add(name); + return this; + } + + public ArgSpec build() { + return new ArgSpec(new LinkedHashMap<>(valueDomains), Set.copyOf(csvArgs)); + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java new file mode 100644 index 00000000000..8433ec5c58b --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +/** + * One column of a {@code rest} endpoint's fixed output schema: a name, a {@link ColumnType}, and a + * {@link RedactionClass} classifying the sensitivity of its values. The schema is fixed so the + * query planner can pin the row type before execution. Column order is the output field order. + * + *

The redaction class is declared once here and reused by every endpoint that has such a column: + * the platform registers a masker per class (OSS registers none), and the {@code rest} choke point + * masks a cell only when its column class is not {@link RedactionClass#NONE} and a masker is + * registered for that class. {@link #of(String, ColumnType)} defaults the class to {@code NONE}. + */ +public record Column(String name, ColumnType type, RedactionClass redaction) { + + public Column { + redaction = redaction == null ? RedactionClass.NONE : redaction; + } + + public static Column of(String name, ColumnType type) { + return new Column(name, type, RedactionClass.NONE); + } + + public static Column of(String name, ColumnType type, RedactionClass redaction) { + return new Column(name, type, redaction); + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java new file mode 100644 index 00000000000..2020d444467 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +/** + * The output type of a {@link Column} in a {@code rest} endpoint schema. + * + *

A deliberately small, dependency-free enum so the SPI does not leak the sql engine's {@code + * ExprType} across the module/classloader boundary. The sql side maps each value onto its own type + * system when it adapts a {@link RestEndpointDefinition} into an internal endpoint. + */ +public enum ColumnType { + STRING, + INTEGER, + LONG, + DOUBLE, + BOOLEAN +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java new file mode 100644 index 00000000000..3090c79bd75 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +/** + * The sensitivity class of a {@link Column}'s values, the join key for {@code rest} response + * redaction. A field is classified ONCE on its column; the platform decides how (or whether) to + * mask each class, so the same masking is reused across every endpoint that declares a column of + * that class, endpoint- and column-name-agnostic. + * + *

Closed and core-owned (never extended by a plugin) so the set of maskable classes is + * auditable. {@link #NONE} is the default and means the value is never redacted. + */ +public enum RedactionClass { + /** Not sensitive; never redacted (the default for a column). */ + NONE, + /** An IP address (IPv4, IPv6, or an {@code inet[/...]} form). */ + IP, + /** A host name (for example an EC2 style {@code ip-a-b-c-d} name). */ + HOSTNAME, + /** An availability-zone name (for example {@code us-east-1a}). */ + AVAILABILITY_ZONE, + /** Free text that may embed any of the above network identifiers. */ + NETWORK_TEXT +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java new file mode 100644 index 00000000000..89889f4e889 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +/** + * Masks the value of a cell whose {@link Column} declares a given {@link RedactionClass}. A + * platform (never a contributed endpoint plugin) registers one {@code Redactor} per class in the + * sql side redaction registry; OSS registers none, so by default nothing is masked. Applied at the + * single {@code rest} row-shaping choke point, once per string cell whose column class is not + * {@link RedactionClass#NONE}. + */ +@FunctionalInterface +public interface Redactor { + + /** + * Return the masked form of one cell value. + * + * @param value the coerced string value of the cell (never null when called from the choke point) + * @return the value to emit in place of {@code value} + */ + String mask(String value); +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java new file mode 100644 index 00000000000..7fcd789ab1a --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.Map; +import org.opensearch.transport.client.node.NodeClient; + +/** + * The per-invocation context handed to a {@link RestEndpointHandler#fetch} at execution time (scan + * open), never at planning time. It carries the validated query args and the node transport client + * a provider may use to issue its own read-only transport action. + * + *

A provider that already holds a client (e.g. the built-in core endpoints capture the sql + * client via closure) may ignore {@link #client()} and read only {@link #args()}. A transport + * backed provider may block on {@code client().execute(action, request).actionGet()} inside {@code + * fetch}; because {@code fetch} runs at execution rather than planning, EXPLAIN stays side-effect + * free. + */ +public interface RestEndpointContext { + + /** The validated query args for this invocation (never null; empty when none were supplied). */ + Map args(); + + /** + * The node transport client, for a provider that issues its own read-only transport action. May + * be null in unit tests or for providers that do not need it. + */ + NodeClient client(); + + static RestEndpointContext of(Map args, NodeClient client) { + Map safeArgs = args == null ? Map.of() : args; + return new RestEndpointContext() { + @Override + public Map args() { + return safeArgs; + } + + @Override + public NodeClient client() { + return client; + } + }; + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java new file mode 100644 index 00000000000..9bfabc3f18d --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; +import java.util.Objects; + +/** + * One read-only {@code rest} endpoint contributed by a {@link RestEndpointProvider}: a unique name + * (the token after {@code rest}, e.g. {@code /_cluster/health}), a fixed output {@link Column} + * schema (so the query planner can pin the row type before execution), the {@link ArgSpec} it + * accepts, and the {@link RestEndpointHandler} that produces its rows at execution time. + * + *

Build one with {@link #builder()}. Instances are immutable. + */ +public interface RestEndpointDefinition { + + String name(); + + List schema(); + + ArgSpec argSpec(); + + RestEndpointHandler handler(); + + static Builder builder() { + return new Builder(); + } + + final class Builder { + private String name; + private List schema = List.of(); + private ArgSpec argSpec = ArgSpec.NONE; + private RestEndpointHandler handler; + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder schema(List schema) { + this.schema = schema; + return this; + } + + public Builder argSpec(ArgSpec argSpec) { + this.argSpec = argSpec; + return this; + } + + public Builder handler(RestEndpointHandler handler) { + this.handler = handler; + return this; + } + + public RestEndpointDefinition build() { + String endpointName = Objects.requireNonNull(name, "rest endpoint name is required"); + List cols = List.copyOf(Objects.requireNonNull(schema, "schema is required")); + ArgSpec spec = Objects.requireNonNull(argSpec, "argSpec is required"); + RestEndpointHandler endpointHandler = Objects.requireNonNull(handler, "handler is required"); + return new RestEndpointDefinition() { + @Override + public String name() { + return endpointName; + } + + @Override + public List schema() { + return cols; + } + + @Override + public ArgSpec argSpec() { + return spec; + } + + @Override + public RestEndpointHandler handler() { + return endpointHandler; + } + }; + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java new file mode 100644 index 00000000000..e613839756e --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; +import java.util.Map; + +/** + * Produces the raw rows of a {@code rest} endpoint. Invoked at execution time (scan open), never at + * planning time, so the scan stays lazy and EXPLAIN is side-effect free. A transport backed + * provider may block on {@code ctx.client().execute(action, request).actionGet()} inside {@link + * #fetch}; because {@code fetch} runs at execution rather than planning, blocking here is expected + * and safe. + * + *

Each row is a map of output-column name to a plain {@code java.lang} value (String, Number, + * Boolean, or a nested Map of the same). The sql side coerces each value to the declared {@link + * Column} type and rejects an uncoercible value with a clear client error, so a handler does not + * need to pre-shape its values to the schema. + */ +@FunctionalInterface +public interface RestEndpointHandler { + + /** + * Fetch the rows for one invocation. + * + * @param ctx the validated query args and (optional) transport client for this invocation + * @return one map of column name to value per row (never null; empty when there are no rows) + */ + List> fetch(RestEndpointContext ctx); +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java new file mode 100644 index 00000000000..b73483f08d0 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; + +/** + * The extension point a plugin implements to contribute read-only {@code rest} endpoints. A plugin + * registers a provider through {@code ExtensiblePlugin.loadExtensions(RestEndpointProvider.class)} + * (a {@code META-INF/services} entry plus {@code extended.plugins=opensearch-sql}); the sql plugin + * merges every discovered provider with its own built-in provider into one registry, so a built-in + * endpoint and an externally contributed endpoint are uniform clients of the same contract. + * + *

A provider declares data (endpoint name, fixed schema, accepted args) and a handler; it never + * touches the PPL grammar. The single {@code rest } command routes to whichever provider + * registered {@code }. + */ +public interface RestEndpointProvider { + + /** The endpoints this provider contributes. Called once when the registry is built. */ + List getEndpoints(); +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index da9424c9c8f..367d1509e4a 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -55,11 +55,12 @@ public void restHealthProjectsDeclaredColumns() { @Test public void restReservedNameRoundTrips() { RestRelation rest = - (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + (RestRelation) + parse("| rest \"/_cluster/health\" count=10 timeout=\"5s\" health=\"green\""); String reserved = rest.getTableQualifiedName().toString(); assertTrue(SystemIndexUtils.isRestSource(reserved)); SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); - assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals("/_cluster/health", spec.getEndpoint()); assertEquals(Integer.valueOf(10), spec.getCount()); assertEquals("5s", spec.getTimeout()); assertEquals("green", spec.getArgs().get("health")); diff --git a/settings.gradle b/settings.gradle index 7fc93fe1725..7546d0093cf 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,6 +17,7 @@ include 'opensearch-sql-plugin' project(':opensearch-sql-plugin').projectDir = file('plugin') include 'api' include 'ppl' +include 'ppl-rest-spi' include 'common' include 'opensearch' include 'core' From ac3f19449d42187de5cfe12b8b11ef89141f7330 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 12:45:36 +0800 Subject: [PATCH 16/23] Remove dead code from the PPL rest framework - ArgSpec: drop the unused csv-arg path (csvArg builder, csvArgs field, csv branch); no endpoint declares a csv arg (leftover from the deferred _cat endpoints). - RedactionRegistry: drop unused isEmpty() (consumers call mask()). - RestRequest: drop the unused 3-arg convenience ctor; fix a stale comment left by the ctx.client() refactor. Signed-off-by: Louis Chu --- .../storage/rest/RedactionRegistry.java | 5 ---- .../opensearch/storage/rest/RestRequest.java | 11 ++----- .../org/opensearch/sql/spi/rest/ArgSpec.java | 30 +++---------------- 3 files changed, 7 insertions(+), 39 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java index 7cce3480cf1..6657347ed57 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java @@ -54,11 +54,6 @@ public void register(RedactionClass redactionClass, Redactor redactor) { redactors.put(redactionClass, redactor); } - /** True when no masker is registered (the OSS default): every cell passes through unchanged. */ - public boolean isEmpty() { - return redactors.isEmpty(); - } - /** * Mask one cell value for its column's class. A {@code null}/{@code NONE} class, a null value, or * a class with no registered masker all pass the value through unchanged. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index 8cfb8f697e3..92e341d2214 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -28,11 +28,6 @@ public class RestRequest implements OpenSearchSystemRequest { private final RestSpec spec; private final RedactionRegistry redaction; - public RestRequest( - OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { - this(client, endpoint, spec, new RedactionRegistry()); - } - public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, @@ -46,9 +41,9 @@ public RestRequest( @Override public List search() { - // The context's transport client is only used by externally contributed providers; the - // built-in provider's handlers hold their own client. Sourced from the same OpenSearchClient - // the storage engine uses, so it runs under the caller's security thread-context. + // The node transport client every provider handler fetches through, sourced from the same + // OpenSearchClient the storage engine uses so it runs under the caller's security + // thread-context. NodeClient nodeClient = client == null ? null : client.getNodeClient().orElse(null); RestEndpointContext ctx = RestEndpointContext.of(spec.getArgs(), nodeClient); List rows = endpoint.toRows(ctx, redaction); diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java index 6e88417792e..30d46bf5aa8 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java @@ -6,7 +6,6 @@ package org.opensearch.sql.spi.rest; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -17,8 +16,7 @@ * declared arg whose value falls outside its domain is rejected. Both checks are enforced uniformly * by the sql registry, so a provider expresses its policy as data rather than as code. * - *

An arg declared with an empty domain accepts any value. An arg declared as a {@code csvArg} - * validates each comma-separated token against the domain (e.g. {@code expand_wildcards}). + *

An arg declared with an empty domain accepts any value. */ public final class ArgSpec { @@ -26,12 +24,9 @@ public final class ArgSpec { // arg name -> allowed values (empty set means "any value allowed"). private final Map> valueDomains; - // subset of arg names whose value is a comma-separated list validated token-by-token. - private final Set csvArgs; - private ArgSpec(Map> valueDomains, Set csvArgs) { + private ArgSpec(Map> valueDomains) { this.valueDomains = valueDomains; - this.csvArgs = csvArgs; } public Set allowedArgs() { @@ -51,16 +46,7 @@ public void validateValue(String endpoint, String arg, String value) { if (domain == null || domain.isEmpty()) { return; } - if (csvArgs.contains(arg)) { - if (value == null || value.isBlank()) { - throw unsupported(endpoint, arg, value, domain); - } - for (String token : value.toLowerCase(Locale.ROOT).split(",")) { - if (!domain.contains(token.trim())) { - throw unsupported(endpoint, arg, value, domain); - } - } - } else if (value == null || !domain.contains(value.toLowerCase(Locale.ROOT))) { + if (value == null || !domain.contains(value.toLowerCase(Locale.ROOT))) { throw unsupported(endpoint, arg, value, domain); } } @@ -85,7 +71,6 @@ public static Builder builder() { /** Builder for {@link ArgSpec}. Declaration order is preserved for stable error messages. */ public static final class Builder { private final LinkedHashMap> valueDomains = new LinkedHashMap<>(); - private final Set csvArgs = new LinkedHashSet<>(); public Builder arg(String name) { valueDomains.put(name, Set.of()); @@ -97,15 +82,8 @@ public Builder arg(String name, Set domain) { return this; } - /** Accept {@code name} as a comma-separated list; every token must be in {@code domain}. */ - public Builder csvArg(String name, Set domain) { - valueDomains.put(name, Set.copyOf(domain)); - csvArgs.add(name); - return this; - } - public ArgSpec build() { - return new ArgSpec(new LinkedHashMap<>(valueDomains), Set.copyOf(csvArgs)); + return new ArgSpec(new LinkedHashMap<>(valueDomains)); } } } From 7637f4da1e6852d042257c4fb50b8ea3ed016436 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 13:51:10 +0800 Subject: [PATCH 17/23] Redaction: replace the per-facet RedactionClass registry with a single centralized Redactor The framework now exposes one Redactor seam ((endpoint, row) -> row, with a NONE no-op) published through RedactorHolder and applied once per raw row at the rest choke point before coercion; the default is NONE. Removes RedactionClass, RedactionRegistry, RedactionRegistryHolder and the per-column redaction tag on Column, so the framework stays field- and facet-agnostic, one coordinated masking pass avoids independent maskers interfering on the same value, and redaction has a single central owner. Signed-off-by: Louis Chu --- .../storage/OpenSearchStorageEngine.java | 12 +-- .../storage/rest/CoreEndpointsProvider.java | 7 +- .../storage/rest/RedactionRegistry.java | 68 ------------ .../storage/rest/RedactionRegistryHolder.java | 31 ------ .../storage/rest/RedactorHolder.java | 34 ++++++ .../storage/rest/RestCatalogSource.java | 14 ++- .../storage/rest/RestEndpointRegistry.java | 39 ++----- .../opensearch/storage/rest/RestRequest.java | 9 +- .../rest/CoreEndpointsProviderTest.java | 3 +- .../storage/rest/RedactionByClassTest.java | 101 ------------------ .../storage/rest/RedactorSeamTest.java | 65 +++++++++++ .../rest/RestEndpointExtensibilityTest.java | 3 +- .../rest/RestEndpointRegistryTest.java | 14 +-- .../org/opensearch/sql/plugin/SQLPlugin.java | 6 +- .../org/opensearch/sql/spi/rest/Column.java | 23 +--- .../sql/spi/rest/RedactionClass.java | 28 ----- .../org/opensearch/sql/spi/rest/Redactor.java | 29 +++-- 17 files changed, 164 insertions(+), 322 deletions(-) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java delete mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 31b36aed17f..5b64442d795 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -17,13 +17,13 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.rest.RedactionRegistry; -import org.opensearch.sql.opensearch.storage.rest.RedactionRegistryHolder; +import org.opensearch.sql.opensearch.storage.rest.RedactorHolder; import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; @@ -71,10 +71,10 @@ private Table restTable(String name) { + "] is not enabled on this cluster. Enabled endpoints: " + allowed); } - // Redaction is a platform-owned control applied by RedactionClass at the row-shaping choke - // point. OSS publishes an empty registry (no-op); a managed distribution fills it via patch. - RedactionRegistry redaction = RedactionRegistryHolder.get(); + // Redaction is applied by the single Redactor at the row-shaping choke point; the default is + // Redactor.NONE (no-op). + Redactor redactor = RedactorHolder.get(); return new OpenSearchCatalogTable( - new RestCatalogSource(registry, spec, client, redaction), settings); + new RestCatalogSource(registry, spec, client, redactor), settings); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java index 7bfb2f21fe1..25f9b036b0c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java @@ -24,12 +24,10 @@ import org.opensearch.transport.client.node.NodeClient; /** - * The built-in {@link RestEndpointProvider}. PR1 ships a single read-only, in-cluster endpoint, + * The built-in {@link RestEndpointProvider}. It ships a single read-only, in-cluster endpoint, * {@code /_cluster/health}, expressed as a {@link RestEndpointDefinition}. It is a uniform client * of the same SPI an external plugin uses; it holds no privileged position in {@link - * RestEndpointRegistry}. The network-bearing endpoints ({@code /_cat/*}, {@code /_cluster/state}, - * {@code /_cluster/settings}, {@code /_resolve/index}, proposed {@code /_nodes/info}) are deferred - * to the AppSec follow-up. + * RestEndpointRegistry}. Additional endpoints are left to follow-up changes. * *

Like any external provider, it fetches at execution time through the transport node client the * context carries ({@link RestEndpointContext#client()}), so it holds no reference to the sql @@ -40,7 +38,6 @@ public final class CoreEndpointsProvider implements RestEndpointProvider { @Override public List getEndpoints() { return List.of( - // /_cluster/health carries no network identifiers, so every column is RedactionClass.NONE. RestEndpointDefinition.builder() .name("/_cluster/health") .schema( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java deleted file mode 100644 index 6657347ed57..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistry.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.EnumMap; -import java.util.Map; -import org.opensearch.sql.spi.rest.RedactionClass; -import org.opensearch.sql.spi.rest.Redactor; - -/** - * The platform-owned map of {@link RedactionClass} to {@link Redactor} that the {@code rest} choke - * point consults when shaping rows. This is deliberately NOT a {@code loadExtensions} SPI: - * redaction is a security control, so an open extension point would let any installed plugin - * register a no-op masker and weaken masking. Instead the registry is populated only by the - * platform through {@link #register} or the constructor. - * - *

OSS leaves it EMPTY, so masking is a pure no-op out of the box. A managed distribution - * supplies per-class {@link Redactor}s through a wrapper patch (precedent: the AOS SQL patch), - * reusing one masker for every endpoint that declares a column of that class. {@link - * RedactionClass#NONE} is a sentinel meaning "never redact" and cannot be registered. - */ -public final class RedactionRegistry { - - private final Map redactors; - - /** An empty registry: every class is a no-op passthrough (the OSS default). */ - public RedactionRegistry() { - this.redactors = new EnumMap<>(RedactionClass.class); - } - - /** A registry pre-populated from a class -> masker map (the managed wrapper seam). */ - public RedactionRegistry(Map redactors) { - this(); - if (redactors != null) { - redactors.forEach(this::register); - } - } - - /** - * Register the masker for one sensitivity class. Rejects {@link RedactionClass#NONE} (the - * never-redact sentinel) and null arguments so the registry stays auditable. - */ - public void register(RedactionClass redactionClass, Redactor redactor) { - if (redactionClass == null || redactionClass == RedactionClass.NONE) { - throw new IllegalArgumentException("cannot register a redactor for redaction class: NONE"); - } - if (redactor == null) { - throw new IllegalArgumentException( - "redactor for redaction class [" + redactionClass + "] must not be null"); - } - redactors.put(redactionClass, redactor); - } - - /** - * Mask one cell value for its column's class. A {@code null}/{@code NONE} class, a null value, or - * a class with no registered masker all pass the value through unchanged. - */ - public String mask(RedactionClass redactionClass, String value) { - if (redactionClass == null || redactionClass == RedactionClass.NONE || value == null) { - return value; - } - Redactor redactor = redactors.get(redactionClass); - return redactor == null ? value : redactor.mask(value); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java deleted file mode 100644 index 8f9894f1e4e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactionRegistryHolder.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -/** - * Bridge for sharing the platform {@link RedactionRegistry} between plugin bootstrap (where the SQL - * plugin publishes it) and {@code OpenSearchStorageEngine.getTable} (which applies it at query - * time). Mirrors {@link RestEndpointRegistryHolder}. - * - *

Defaults to an empty registry so the OSS choke point is a pure no-op even before bootstrap - * publishes one, and so {@code getTable} never has to null-check. A managed distribution replaces - * or fills it via a wrapper patch that calls {@link #set} (or {@link RedactionRegistry#register}) - * during {@code createComponents}. - */ -public final class RedactionRegistryHolder { - - private static volatile RedactionRegistry registry = new RedactionRegistry(); - - private RedactionRegistryHolder() {} - - public static void set(RedactionRegistry instance) { - registry = instance == null ? new RedactionRegistry() : instance; - } - - public static RedactionRegistry get() { - return registry; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java new file mode 100644 index 00000000000..5864ffc21e9 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.sql.spi.rest.Redactor; + +/** + * Bridge for sharing the platform {@link Redactor} between plugin bootstrap (where the SQL plugin + * publishes it) and {@code OpenSearchStorageEngine.getTable} (which applies it at query time). + * Mirrors {@link RestEndpointRegistryHolder}. + * + *

Defaults to {@link Redactor#NONE} so the OSS choke point is a pure no-op even before bootstrap + * runs and so {@code getTable} never has to null-check; a provider calls {@link #set} during + * {@code createComponents} to install its own implementation. This is intentionally a single + * central hook, not a {@code loadExtensions} SPI, so redaction has one owner rather than an open + * registration surface. + */ +public final class RedactorHolder { + + private static volatile Redactor redactor = Redactor.NONE; + + private RedactorHolder() {} + + public static void set(Redactor instance) { + redactor = instance == null ? Redactor.NONE : instance; + } + + public static Redactor get() { + return redactor; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java index d0d0d2dff99..66b5bacb333 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -13,6 +13,7 @@ import org.opensearch.sql.opensearch.storage.system.CatalogSource; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** @@ -26,20 +27,17 @@ public class RestCatalogSource implements CatalogSource { private final OpenSearchClient client; private final RestSpec spec; private final RestEndpointRegistry.Endpoint endpoint; - private final RedactionRegistry redaction; + private final Redactor redactor; public RestCatalogSource(RestEndpointRegistry registry, RestSpec spec, OpenSearchClient client) { - this(registry, spec, client, new RedactionRegistry()); + this(registry, spec, client, Redactor.NONE); } public RestCatalogSource( - RestEndpointRegistry registry, - RestSpec spec, - OpenSearchClient client, - RedactionRegistry redaction) { + RestEndpointRegistry registry, RestSpec spec, OpenSearchClient client, Redactor redactor) { this.client = client; this.spec = spec; - this.redaction = redaction; + this.redactor = redactor; // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. this.endpoint = registry.resolve(spec.getEndpoint()); registry.validate(spec); @@ -52,7 +50,7 @@ public Map getFieldTypes() { @Override public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec, redaction); + return new RestRequest(client, endpoint, spec, redactor); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index 95da2bcae76..250d71c81cf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -28,7 +28,7 @@ import org.opensearch.sql.spi.rest.ArgSpec; import org.opensearch.sql.spi.rest.Column; import org.opensearch.sql.spi.rest.ColumnType; -import org.opensearch.sql.spi.rest.RedactionClass; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.spi.rest.RestEndpointDefinition; import org.opensearch.sql.spi.rest.RestEndpointHandler; @@ -75,50 +75,33 @@ public java.util.Set endpointNames() { public static final class Endpoint { private final String path; private final LinkedHashMap schema; - // Column name -> sensitivity class, the join key the choke point uses to pick a Redactor. - private final Map redactionClasses; private final ArgSpec argSpec; private final RestEndpointHandler handler; Endpoint(RestEndpointDefinition definition) { this.path = definition.name(); this.schema = toExprSchema(definition.schema()); - this.redactionClasses = toRedactionClasses(definition.schema()); this.argSpec = definition.argSpec(); this.handler = definition.handler(); } /** - * Invoke the provider's handler and shape the response into fixed-schema rows, masking each - * cell by its column's {@link RedactionClass} through the platform {@link RedactionRegistry}. - * Runs at execution time (scan open). An empty registry (the OSS default) masks nothing. + * Invoke the provider's handler, apply the platform {@link Redactor} to each raw row (with the + * endpoint name as scope), then shape it into fixed-schema rows. Runs at execution time (scan + * open). {@link Redactor#NONE} (the OSS default) passes rows through unchanged. */ - public List toRows(RestEndpointContext ctx, RedactionRegistry redaction) { + public List toRows(RestEndpointContext ctx, Redactor redactor) { List out = new ArrayList<>(); for (Map raw : handler.fetch(ctx)) { + Map redacted = redactor.redact(path, raw); LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { - ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); - tuple.put(col.getKey(), maskCell(col.getKey(), col.getValue(), value, redaction)); + tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), redacted.get(col.getKey()))); } out.add(new ExprTupleValue(tuple)); } return out; } - - private ExprValue maskCell( - String column, ExprType type, ExprValue value, RedactionRegistry redaction) { - // Redaction applies only to non-null string cells; a class declared on a non-string column - // (unusual) is ignored so a number is never fed to a text masker. - if (type != STRING || value.isNull()) { - return value; - } - RedactionClass redactionClass = redactionClasses.getOrDefault(column, RedactionClass.NONE); - if (redactionClass == RedactionClass.NONE) { - return value; - } - return stringValue(redaction.mask(redactionClass, value.stringValue())); - } } /** @@ -186,14 +169,6 @@ private static LinkedHashMap toExprSchema(List columns return schema; } - private static Map toRedactionClasses(List columns) { - LinkedHashMap classes = new LinkedHashMap<>(); - for (Column column : columns) { - classes.put(column.name(), column.redaction()); - } - return classes; - } - private static ExprType toExprType(ColumnType type) { return switch (type) { case STRING -> STRING; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index 92e341d2214..1c2036380d1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -9,6 +9,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; import org.opensearch.transport.client.node.NodeClient; @@ -26,17 +27,17 @@ public class RestRequest implements OpenSearchSystemRequest { private final OpenSearchClient client; private final RestEndpointRegistry.Endpoint endpoint; private final RestSpec spec; - private final RedactionRegistry redaction; + private final Redactor redactor; public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec, - RedactionRegistry redaction) { + Redactor redactor) { this.client = client; this.endpoint = endpoint; this.spec = spec; - this.redaction = redaction; + this.redactor = redactor; } @Override @@ -46,7 +47,7 @@ public List search() { // thread-context. NodeClient nodeClient = client == null ? null : client.getNodeClient().orElse(null); RestEndpointContext ctx = RestEndpointContext.of(spec.getArgs(), nodeClient); - List rows = endpoint.toRows(ctx, redaction); + List rows = endpoint.toRows(ctx, redactor); if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { return rows.subList(0, spec.getCount()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java index 2ba6b6707e9..cb36a3253fb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java @@ -19,6 +19,7 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.transport.client.node.NodeClient; @@ -51,7 +52,7 @@ void clusterHealthFetchesViaContextNodeClient() { List rows = registry .resolve("/_cluster/health") - .toRows(RestEndpointContext.of(Map.of(), nodeClient), new RedactionRegistry()); + .toRows(RestEndpointContext.of(Map.of(), nodeClient), Redactor.NONE); assertEquals(1, rows.size()); Map row = rows.get(0).tupleValue(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java deleted file mode 100644 index 3c6f7766d47..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactionByClassTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.spi.rest.ColumnType.STRING; - -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.spi.rest.ArgSpec; -import org.opensearch.sql.spi.rest.Column; -import org.opensearch.sql.spi.rest.RedactionClass; -import org.opensearch.sql.spi.rest.RestEndpointContext; -import org.opensearch.sql.spi.rest.RestEndpointDefinition; -import org.opensearch.sql.spi.rest.RestEndpointProvider; - -/** - * Proves the core property of class-based redaction: a sensitivity class is declared ONCE on a - * column and one platform {@link org.opensearch.sql.spi.rest.Redactor} registered for that class is - * reused by EVERY endpoint that has such a column, endpoint- and column-name-agnostic. Two - * unrelated fake endpoints, each with an {@link RedactionClass#IP} column of a different name, are - * both masked by the single registered IP redactor; a column of another class is untouched; and an - * empty registry (the OSS default) masks nothing. - */ -class RedactionByClassTest { - - private static final String MASK = "x.x.x.x"; - - /** - * Two unrelated endpoints, each with an IP-classed column (different names) plus a NONE column. - */ - private static final class TwoIpEndpointsProvider implements RestEndpointProvider { - @Override - public List getEndpoints() { - return List.of( - RestEndpointDefinition.builder() - .name("/_fake/alpha") - .schema( - List.of( - Column.of("addr", STRING, RedactionClass.IP), - Column.of("label", STRING, RedactionClass.NONE))) - .argSpec(ArgSpec.NONE) - .handler(ctx -> List.of(Map.of("addr", "10.0.0.7", "label", "10.0.0.7"))) - .build(), - RestEndpointDefinition.builder() - .name("/_fake/beta") - .schema( - List.of( - Column.of("host_ip", STRING, RedactionClass.IP), Column.of("note", STRING))) - .argSpec(ArgSpec.NONE) - .handler(ctx -> List.of(Map.of("host_ip", "192.168.1.1", "note", "192.168.1.1"))) - .build()); - } - } - - private static RestEndpointRegistry registry() { - return new RestEndpointRegistry(List.of(new TwoIpEndpointsProvider())); - } - - private static RestEndpointContext ctx() { - return RestEndpointContext.of(Map.of(), null); - } - - @Test - void oneRedactorMasksTheIpColumnOfEveryEndpoint() { - // Declare the IP masker ONCE; it must apply to both endpoints' IP columns regardless of name. - RedactionRegistry redaction = new RedactionRegistry(); - redaction.register(RedactionClass.IP, value -> MASK); - - RestEndpointRegistry registry = registry(); - - Map alpha = - registry.resolve("/_fake/alpha").toRows(ctx(), redaction).get(0).tupleValue(); - assertEquals(MASK, alpha.get("addr").stringValue(), "alpha IP column masked"); - // A NONE-class column is never masked, even though its value is an address. - assertEquals("10.0.0.7", alpha.get("label").stringValue(), "NONE-class column untouched"); - - Map beta = - registry.resolve("/_fake/beta").toRows(ctx(), redaction).get(0).tupleValue(); - assertEquals( - MASK, beta.get("host_ip").stringValue(), "beta IP column masked by the same redactor"); - assertEquals("192.168.1.1", beta.get("note").stringValue(), "default NONE column untouched"); - } - - @Test - void emptyRegistryIsANoOp() { - RedactionRegistry empty = new RedactionRegistry(); - RestEndpointRegistry registry = registry(); - - Map alpha = - registry.resolve("/_fake/alpha").toRows(ctx(), empty).get(0).tupleValue(); - // No redactor registered for IP: the value passes through unchanged. - assertEquals("10.0.0.7", alpha.get("addr").stringValue()); - assertEquals("10.0.0.7", alpha.get("label").stringValue()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java new file mode 100644 index 00000000000..e2f57fe4532 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.ColumnType; +import org.opensearch.sql.spi.rest.Redactor; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; + +/** + * Proves the {@code rest} choke point applies the single {@link Redactor} to each raw response + * row (with the endpoint name as scope) before coercion, and that {@link Redactor#NONE} passes + * rows through unchanged. + */ +class RedactorSeamTest { + + private static RestEndpointRegistry.Endpoint probe() { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_test/probe") + .schema( + List.of( + Column.of("ip", ColumnType.STRING), + Column.of("count", ColumnType.INTEGER))) + .handler(ctx -> List.of(Map.of("ip", "10.0.0.7", "count", 3))) + .build()); + return new RestEndpointRegistry(List.of(provider)).resolve("/_test/probe"); + } + + @Test + void chokePointAppliesRedactorToRawRowBeforeCoercion() { + Redactor maskIp = + (endpoint, row) -> { + Map masked = new HashMap<>(row); + masked.put("ip", "x.x.x.x"); + return masked; + }; + + List rows = probe().toRows(RestEndpointContext.of(Map.of(), null), maskIp); + + assertEquals("x.x.x.x", rows.get(0).tupleValue().get("ip").stringValue()); + // A column the redactor did not touch is still coerced to its schema type. + assertEquals(3, rows.get(0).tupleValue().get("count").integerValue()); + } + + @Test + void noneRedactorPassesThrough() { + List rows = probe().toRows(RestEndpointContext.of(Map.of(), null), Redactor.NONE); + assertEquals("10.0.0.7", rows.get(0).tupleValue().get("ip").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java index 00194ad9b0f..e07217af00e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java @@ -16,6 +16,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.spi.rest.ArgSpec; import org.opensearch.sql.spi.rest.Column; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.spi.rest.RestEndpointDefinition; import org.opensearch.sql.spi.rest.RestEndpointProvider; @@ -63,7 +64,7 @@ void bothBuiltInAndExternalEndpointsResolve() { void externalEndpointFetchesThroughTheSamePath() { RestEndpointRegistry.Endpoint echo = mergedRegistry().resolve("/_plugin/echo"); List rows = - echo.toRows(RestEndpointContext.of(Map.of("text", "hi"), null), new RedactionRegistry()); + echo.toRows(RestEndpointContext.of(Map.of("text", "hi"), null), Redactor.NONE); assertEquals(1, rows.size()); assertEquals("hi", rows.get(0).tupleValue().get("message").stringValue()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index 179bc2f2747..fc93877a5b4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -19,6 +19,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.spi.rest.Column; import org.opensearch.sql.spi.rest.ColumnType; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointContext; import org.opensearch.sql.spi.rest.RestEndpointDefinition; import org.opensearch.sql.spi.rest.RestEndpointProvider; @@ -28,9 +29,9 @@ * Covers the {@code rest} {@link RestEndpointRegistry} against the PR1 endpoint set (only {@code * /_cluster/health}): allow-list resolution, arg validation, count/timeout gating, value coercion, * and fixed-schema row shaping. Coercion and shaping are exercised through a fake provider that - * returns canned rows, so they stay independent of how any one endpoint fetches. Redaction-by-class - * is covered separately in {@link RedactionByClassTest}; here rows are shaped with an empty {@link - * RedactionRegistry} (no-op). + * returns canned rows, so they stay independent of how any one endpoint fetches; rows are shaped + * with {@link Redactor#NONE} (the no-op redactor). The centralized redactor seam is covered + * separately in {@link RedactorSeamTest}. */ class RestEndpointRegistryTest { @@ -146,7 +147,7 @@ void coerceParsesNumericStringValues() { Column.of("status", ColumnType.STRING), Column.of("number_of_nodes", ColumnType.INTEGER)), Map.of("status", "green", "number_of_nodes", "3")); - List rows = endpoint.toRows(ctx(Map.of()), new RedactionRegistry()); + List rows = endpoint.toRows(ctx(Map.of()), Redactor.NONE); assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); } @@ -162,8 +163,7 @@ void coerceThrowsClearErrorOnUncoercibleValue() { Map.of("status", "green", "number_of_nodes", "not-a-number")); IllegalArgumentException ex = assertThrows( - IllegalArgumentException.class, - () -> endpoint.toRows(ctx(Map.of()), new RedactionRegistry())); + IllegalArgumentException.class, () -> endpoint.toRows(ctx(Map.of()), Redactor.NONE)); assertTrue(ex.getMessage().contains("number_of_nodes")); assertTrue(ex.getMessage().contains("not-a-number")); } @@ -182,7 +182,7 @@ void rowsAreShapedToFixedSchema() { Column.of("number_of_nodes", ColumnType.INTEGER), Column.of("relocating_shards", ColumnType.INTEGER)), raw); - List rows = endpoint.toRows(ctx(Map.of()), new RedactionRegistry()); + List rows = endpoint.toRows(ctx(Map.of()), Redactor.NONE); assertEquals(1, rows.size()); assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index c88f8003b15..a62432bbd5f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -105,8 +105,7 @@ import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; -import org.opensearch.sql.opensearch.storage.rest.RedactionRegistry; -import org.opensearch.sql.opensearch.storage.rest.RedactionRegistryHolder; +import org.opensearch.sql.opensearch.storage.rest.RedactorHolder; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; @@ -140,6 +139,7 @@ import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.CreateAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse; +import org.opensearch.sql.spi.rest.Redactor; import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.storage.DataSourceFactory; @@ -199,7 +199,7 @@ private void publishRestCommandRegistries() { providers.add(new CoreEndpointsProvider()); providers.addAll(this.restEndpointProviders); RestEndpointRegistryHolder.set(new RestEndpointRegistry(providers)); - RedactionRegistryHolder.set(new RedactionRegistry()); + RedactorHolder.set(Redactor.NONE); } @Override diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java index 8433ec5c58b..3e249eecf74 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java @@ -6,26 +6,13 @@ package org.opensearch.sql.spi.rest; /** - * One column of a {@code rest} endpoint's fixed output schema: a name, a {@link ColumnType}, and a - * {@link RedactionClass} classifying the sensitivity of its values. The schema is fixed so the - * query planner can pin the row type before execution. Column order is the output field order. - * - *

The redaction class is declared once here and reused by every endpoint that has such a column: - * the platform registers a masker per class (OSS registers none), and the {@code rest} choke point - * masks a cell only when its column class is not {@link RedactionClass#NONE} and a masker is - * registered for that class. {@link #of(String, ColumnType)} defaults the class to {@code NONE}. + * One column of a {@code rest} endpoint's fixed output schema: a name and a {@link ColumnType}. The + * schema is fixed so the query planner can pin the row type before execution. Column order is the + * output field order. */ -public record Column(String name, ColumnType type, RedactionClass redaction) { - - public Column { - redaction = redaction == null ? RedactionClass.NONE : redaction; - } +public record Column(String name, ColumnType type) { public static Column of(String name, ColumnType type) { - return new Column(name, type, RedactionClass.NONE); - } - - public static Column of(String name, ColumnType type, RedactionClass redaction) { - return new Column(name, type, redaction); + return new Column(name, type); } } diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java deleted file mode 100644 index 3090c79bd75..00000000000 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RedactionClass.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.spi.rest; - -/** - * The sensitivity class of a {@link Column}'s values, the join key for {@code rest} response - * redaction. A field is classified ONCE on its column; the platform decides how (or whether) to - * mask each class, so the same masking is reused across every endpoint that declares a column of - * that class, endpoint- and column-name-agnostic. - * - *

Closed and core-owned (never extended by a plugin) so the set of maskable classes is - * auditable. {@link #NONE} is the default and means the value is never redacted. - */ -public enum RedactionClass { - /** Not sensitive; never redacted (the default for a column). */ - NONE, - /** An IP address (IPv4, IPv6, or an {@code inet[/...]} form). */ - IP, - /** A host name (for example an EC2 style {@code ip-a-b-c-d} name). */ - HOSTNAME, - /** An availability-zone name (for example {@code us-east-1a}). */ - AVAILABILITY_ZONE, - /** Free text that may embed any of the above network identifiers. */ - NETWORK_TEXT -} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java index 89889f4e889..5bb9d2cdaef 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java @@ -5,21 +5,32 @@ package org.opensearch.sql.spi.rest; +import java.util.Map; + /** - * Masks the value of a cell whose {@link Column} declares a given {@link RedactionClass}. A - * platform (never a contributed endpoint plugin) registers one {@code Redactor} per class in the - * sql side redaction registry; OSS registers none, so by default nothing is masked. Applied at the - * single {@code rest} row-shaping choke point, once per string cell whose column class is not - * {@link RedactionClass#NONE}. + * The single central redaction hook for the {@code rest} command. It receives one raw + * response row (column name to value) for a given endpoint and returns the row to emit, with any + * sensitive values masked in one coordinated pass. + * + *

This is deliberately NOT a {@code loadExtensions} SPI and NOT a per-field classification: + * masking has one central owner rather than an open registration surface. The default is {@link + * #NONE} (a no-op). Because one implementation owns the whole row, it decides its own masking order + * and never has independent maskers interfering on the same value. The endpoint name is passed as a + * scope so the implementation can vary masking per endpoint. Applied once per row at the {@code + * rest} row-shaping choke point, before values are coerced to the fixed schema types. */ @FunctionalInterface public interface Redactor { + /** A no-op that returns the row unchanged (the OSS default). */ + Redactor NONE = (endpoint, row) -> row; + /** - * Return the masked form of one cell value. + * Redact one raw response row. * - * @param value the coerced string value of the cell (never null when called from the choke point) - * @return the value to emit in place of {@code value} + * @param endpoint the endpoint name the row came from, a scope for endpoint-aware masking + * @param row column name to raw value for one row (never null) + * @return the row to emit, with sensitive values masked */ - String mask(String value); + Map redact(String endpoint, Map row); } From 7b63421146b95da2379f70dcb17094d01e2298e6 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 16:39:21 +0800 Subject: [PATCH 18/23] Trim rest javadoc/comments for concision; mark rest 3.9 in the PPL command index Shorten the ppl-rest-spi javadoc and the visitRestCommand comment to a concise, consistent style, correct the rest.md endpoint note, and list the rest command as 3.9 (experimental) in the PPL command index. Signed-off-by: Louis Chu --- docs/user/ppl/cmd/rest.md | 2 +- docs/user/ppl/index.md | 2 +- .../org/opensearch/sql/spi/rest/ArgSpec.java | 14 ++++---------- .../org/opensearch/sql/spi/rest/Column.java | 5 ++--- .../opensearch/sql/spi/rest/ColumnType.java | 7 ++----- .../org/opensearch/sql/spi/rest/Redactor.java | 15 +++++---------- .../sql/spi/rest/RestEndpointContext.java | 19 +++++-------------- .../sql/spi/rest/RestEndpointDefinition.java | 10 ++++------ .../sql/spi/rest/RestEndpointHandler.java | 13 ++++--------- .../sql/spi/rest/RestEndpointProvider.java | 14 +++++--------- .../opensearch/sql/ppl/parser/AstBuilder.java | 9 +++------ 11 files changed, 36 insertions(+), 74 deletions(-) diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index e2cf098fa7d..fdbc9fc7935 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -4,7 +4,7 @@ The `rest` command is a leading command that reads an allow-listed, read-only in > **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. -The `rest` command is a generic, extensible framework: a plugin contributes additional read-only endpoints through the `RestEndpointProvider` extension point without changing the grammar. This first version ships a single built-in endpoint, `/_cluster/health`. Endpoints that surface network topology or other sensitive metadata (for example `/_cat/nodes`, `/_cat/shards`, `/_cluster/state`, `/_cluster/settings`) are gated behind a security review and are added subsequently, together with schema-declared response redaction (a `RedactionClass` per column, masked by a platform-owned redactor). +The `rest` command is a generic, extensible framework: a plugin contributes additional read-only endpoints through the `RestEndpointProvider` extension point without changing the grammar. This first version ships a single built-in endpoint, `/_cluster/health`. Additional endpoints (for example `/_cat/nodes`, `/_cat/shards`, `/_cluster/state`, `/_cluster/settings`) can be added in follow-ups, together with optional response redaction applied centrally at the endpoint choke point. ## Enabling the command diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 24d6aa64c67..3e5a08d990d 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -75,7 +75,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | +| [rest command](cmd/rest.md) | 3.9 | experimental (since 3.9) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java index 30d46bf5aa8..003c56cca6a 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java @@ -11,12 +11,9 @@ import java.util.Set; /** - * The query args a {@code rest} endpoint accepts, and the allowed value domain of each. This is the - * per-endpoint half of the allow-list: an arg the endpoint does not declare is rejected, and a - * declared arg whose value falls outside its domain is rejected. Both checks are enforced uniformly - * by the sql registry, so a provider expresses its policy as data rather than as code. - * - *

An arg declared with an empty domain accepts any value. + * The query args a {@code rest} endpoint accepts, with the allowed value domain of each. An + * undeclared arg is rejected; a declared arg whose value is outside its domain is rejected; an + * empty domain accepts any value. */ public final class ArgSpec { @@ -37,10 +34,7 @@ public boolean allows(String arg) { return valueDomains.containsKey(arg); } - /** - * Validate the value of an accepted arg against its domain. No-op if the arg has no domain (any - * value allowed). Throws {@link IllegalArgumentException} with a clear client error otherwise. - */ + /** Validate an accepted arg's value against its domain (no-op if the domain is empty). */ public void validateValue(String endpoint, String arg, String value) { Set domain = valueDomains.get(arg); if (domain == null || domain.isEmpty()) { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java index 3e249eecf74..da91ed6ab0c 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Column.java @@ -6,9 +6,8 @@ package org.opensearch.sql.spi.rest; /** - * One column of a {@code rest} endpoint's fixed output schema: a name and a {@link ColumnType}. The - * schema is fixed so the query planner can pin the row type before execution. Column order is the - * output field order. + * One column of a {@code rest} endpoint's fixed output schema: a name and a {@link ColumnType}. + * Column order is output-field order. */ public record Column(String name, ColumnType type) { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java index 2020d444467..b3f0cf7c2cf 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ColumnType.java @@ -6,11 +6,8 @@ package org.opensearch.sql.spi.rest; /** - * The output type of a {@link Column} in a {@code rest} endpoint schema. - * - *

A deliberately small, dependency-free enum so the SPI does not leak the sql engine's {@code - * ExprType} across the module/classloader boundary. The sql side maps each value onto its own type - * system when it adapts a {@link RestEndpointDefinition} into an internal endpoint. + * The output type of a {@link Column}. A small, dependency-free enum so the SPI does not leak the + * sql engine's {@code ExprType} across the module boundary. */ public enum ColumnType { STRING, diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java index 5bb9d2cdaef..6773a9ed416 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/Redactor.java @@ -8,16 +8,11 @@ import java.util.Map; /** - * The single central redaction hook for the {@code rest} command. It receives one raw - * response row (column name to value) for a given endpoint and returns the row to emit, with any - * sensitive values masked in one coordinated pass. - * - *

This is deliberately NOT a {@code loadExtensions} SPI and NOT a per-field classification: - * masking has one central owner rather than an open registration surface. The default is {@link - * #NONE} (a no-op). Because one implementation owns the whole row, it decides its own masking order - * and never has independent maskers interfering on the same value. The endpoint name is passed as a - * scope so the implementation can vary masking per endpoint. Applied once per row at the {@code - * rest} row-shaping choke point, before values are coerced to the fixed schema types. + * The single redaction hook for the {@code rest} command: given one raw response row (column name + * to value) for an endpoint, it returns the row to emit with sensitive values masked in one pass. + * Not a {@code loadExtensions} SPI (one central owner, not an open registration surface); the + * default is {@link #NONE} (no-op), applied once per row at the choke point before values are + * coerced to the schema types. */ @FunctionalInterface public interface Redactor { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java index 7fcd789ab1a..3cf78911deb 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java @@ -9,25 +9,16 @@ import org.opensearch.transport.client.node.NodeClient; /** - * The per-invocation context handed to a {@link RestEndpointHandler#fetch} at execution time (scan - * open), never at planning time. It carries the validated query args and the node transport client - * a provider may use to issue its own read-only transport action. - * - *

A provider that already holds a client (e.g. the built-in core endpoints capture the sql - * client via closure) may ignore {@link #client()} and read only {@link #args()}. A transport - * backed provider may block on {@code client().execute(action, request).actionGet()} inside {@code - * fetch}; because {@code fetch} runs at execution rather than planning, EXPLAIN stays side-effect - * free. + * Per-invocation context handed to {@link RestEndpointHandler#fetch} at execution time (scan open), + * carrying the validated query args and the node transport client a provider may use for its own + * read-only transport action. A provider that already holds a client may ignore {@link #client()}. */ public interface RestEndpointContext { - /** The validated query args for this invocation (never null; empty when none were supplied). */ + /** Validated query args for this invocation (never null; empty when none supplied). */ Map args(); - /** - * The node transport client, for a provider that issues its own read-only transport action. May - * be null in unit tests or for providers that do not need it. - */ + /** Node transport client for a provider that issues its own transport action; may be null. */ NodeClient client(); static RestEndpointContext of(Map args, NodeClient client) { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java index 9bfabc3f18d..a36262db9a1 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java @@ -9,12 +9,10 @@ import java.util.Objects; /** - * One read-only {@code rest} endpoint contributed by a {@link RestEndpointProvider}: a unique name - * (the token after {@code rest}, e.g. {@code /_cluster/health}), a fixed output {@link Column} - * schema (so the query planner can pin the row type before execution), the {@link ArgSpec} it - * accepts, and the {@link RestEndpointHandler} that produces its rows at execution time. - * - *

Build one with {@link #builder()}. Instances are immutable. + * One read-only {@code rest} endpoint from a {@link RestEndpointProvider}: a unique name (the token + * after {@code rest}, e.g. {@code /_cluster/health}), a fixed {@link Column} schema, the {@link + * ArgSpec} it accepts, and the {@link RestEndpointHandler} that produces its rows. Immutable; build + * with {@link #builder()}. */ public interface RestEndpointDefinition { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java index e613839756e..ca59fab727c 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java @@ -10,15 +10,10 @@ /** * Produces the raw rows of a {@code rest} endpoint. Invoked at execution time (scan open), never at - * planning time, so the scan stays lazy and EXPLAIN is side-effect free. A transport backed - * provider may block on {@code ctx.client().execute(action, request).actionGet()} inside {@link - * #fetch}; because {@code fetch} runs at execution rather than planning, blocking here is expected - * and safe. - * - *

Each row is a map of output-column name to a plain {@code java.lang} value (String, Number, - * Boolean, or a nested Map of the same). The sql side coerces each value to the declared {@link - * Column} type and rejects an uncoercible value with a clear client error, so a handler does not - * need to pre-shape its values to the schema. + * planning, so the scan stays lazy and EXPLAIN is side-effect free; a transport-backed provider may + * block on {@code ctx.client().execute(...).actionGet()} here. Each row is a map of output-column + * name to a plain {@code java.lang} value; the sql side coerces each to the declared {@link Column} + * type. */ @FunctionalInterface public interface RestEndpointHandler { diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java index b73483f08d0..db04c016f8f 100644 --- a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java @@ -8,15 +8,11 @@ import java.util.List; /** - * The extension point a plugin implements to contribute read-only {@code rest} endpoints. A plugin - * registers a provider through {@code ExtensiblePlugin.loadExtensions(RestEndpointProvider.class)} - * (a {@code META-INF/services} entry plus {@code extended.plugins=opensearch-sql}); the sql plugin - * merges every discovered provider with its own built-in provider into one registry, so a built-in - * endpoint and an externally contributed endpoint are uniform clients of the same contract. - * - *

A provider declares data (endpoint name, fixed schema, accepted args) and a handler; it never - * touches the PPL grammar. The single {@code rest } command routes to whichever provider - * registered {@code }. + * The extension point a plugin implements to contribute read-only {@code rest} endpoints, via + * {@code ExtensiblePlugin.loadExtensions(RestEndpointProvider.class)}. The sql plugin merges every + * discovered provider with its own built-in one into a single registry, so built-in and external + * endpoints are uniform clients of the same contract. A provider declares data (name, schema, args) + * and a handler; it never touches the PPL grammar. */ public interface RestEndpointProvider { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index de0270cac9b..7aca48a4666 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -264,12 +264,9 @@ public UnresolvedPlan visitShowDataSourcesCommand( /** * Rest command.
- * Leading command that reads an allow-listed, read-only in-cluster management endpoint - * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved - * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves - * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE - * resolves to a system index. Allow-list/authorization enforcement happens at source-table - * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + * Encodes the validated endpoint spec into a reserved table name (via {@link + * org.opensearch.sql.utils.SystemIndexUtils#restTable}) that resolves through the storage engine + * to a REST source table on the Calcite path, mirroring how DESCRIBE resolves to a system index. */ @Override public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { From 65d3f86d32724df6716816d0b0a38c89c6b9b1e7 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 17:08:29 +0800 Subject: [PATCH 19/23] Apply spotless formatting to rest javadoc Signed-off-by: Louis Chu --- .../sql/opensearch/storage/rest/RedactorHolder.java | 8 ++++---- .../sql/opensearch/storage/rest/RedactorSeamTest.java | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java index 5864ffc21e9..9b7a7b10958 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RedactorHolder.java @@ -13,10 +13,10 @@ * Mirrors {@link RestEndpointRegistryHolder}. * *

Defaults to {@link Redactor#NONE} so the OSS choke point is a pure no-op even before bootstrap - * runs and so {@code getTable} never has to null-check; a provider calls {@link #set} during - * {@code createComponents} to install its own implementation. This is intentionally a single - * central hook, not a {@code loadExtensions} SPI, so redaction has one owner rather than an open - * registration surface. + * runs and so {@code getTable} never has to null-check; a provider calls {@link #set} during {@code + * createComponents} to install its own implementation. This is intentionally a single central hook, + * not a {@code loadExtensions} SPI, so redaction has one owner rather than an open registration + * surface. */ public final class RedactorHolder { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java index e2f57fe4532..1f134486a02 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RedactorSeamTest.java @@ -20,9 +20,9 @@ import org.opensearch.sql.spi.rest.RestEndpointProvider; /** - * Proves the {@code rest} choke point applies the single {@link Redactor} to each raw response - * row (with the endpoint name as scope) before coercion, and that {@link Redactor#NONE} passes - * rows through unchanged. + * Proves the {@code rest} choke point applies the single {@link Redactor} to each raw response row + * (with the endpoint name as scope) before coercion, and that {@link Redactor#NONE} passes rows + * through unchanged. */ class RedactorSeamTest { From 8d371a8b990b968c61db5c842672bcaca3c40837 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 17:45:04 +0800 Subject: [PATCH 20/23] rest: skip a duplicate endpoint name with a WARN instead of failing startup If two providers register the same endpoint name, keep the first registration and log a warning for the duplicate instead of throwing during createComponents. This bounds a naming collision to the offending endpoint rather than failing plugin/node bootstrap, and the first registration still wins so a later provider cannot replace it. Signed-off-by: Louis Chu --- .../storage/rest/RestEndpointRegistry.java | 13 +++++++++---- .../rest/RestEndpointExtensibilityTest.java | 17 ++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index 250d71c81cf..d72011c4966 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -21,6 +21,8 @@ import java.util.List; import java.util.Map; import lombok.Getter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -48,6 +50,8 @@ */ public final class RestEndpointRegistry { + private static final Logger LOG = LogManager.getLogger(RestEndpointRegistry.class); + private final Map registry; public RestEndpointRegistry(List providers) { @@ -55,10 +59,11 @@ public RestEndpointRegistry(List providers) { for (RestEndpointProvider provider : providers) { for (RestEndpointDefinition definition : provider.getEndpoints()) { if (m.containsKey(definition.name())) { - throw new IllegalStateException( - "rest endpoint [" - + definition.name() - + "] is registered by more than one provider; endpoint names must be unique"); + LOG.warn( + "rest endpoint [{}] is already registered; ignoring the duplicate from provider [{}]", + definition.name(), + provider.getClass().getName()); + continue; } m.put(definition.name(), new Endpoint(definition)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java index e07217af00e..f8db570cb29 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.opensearch.storage.rest; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.spi.rest.ColumnType.STRING; @@ -85,19 +86,21 @@ void externalEndpointArgsValidatedByTheSameAllowList() { } @Test - void duplicateEndpointNameAcrossProvidersIsRejected() { + void duplicateEndpointNameAcrossProvidersIsSkippedFirstWins() { + // A second provider that re-declares /_cluster/health is ignored rather than failing the + // build; the first (built-in) registration wins. RestEndpointProvider shadowsCore = () -> List.of( RestEndpointDefinition.builder() .name("/_cluster/health") - .schema(List.of(Column.of("x", STRING))) + .schema(List.of(Column.of("shadow", STRING))) .handler(ctx -> List.of()) .build()); - IllegalStateException ex = - assertThrows( - IllegalStateException.class, - () -> new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadowsCore))); - assertTrue(ex.getMessage().contains("more than one provider")); + RestEndpointRegistry registry = + new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadowsCore)); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertFalse(registry.resolve("/_cluster/health").getSchema().containsKey("shadow")); } } From 19818d41970648589afc4fbd11dac74a5c1fa2c9 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 18:06:03 +0800 Subject: [PATCH 21/23] rest: trim stale test allow-list settings to the one registered endpoint The doctest, integTest, and RestCommandSecurityIT clusters allow-listed endpoints that are not registered in this version; only /_cluster/health exists (and is the default). Trim each to /_cluster/health and correct the comments, which wrongly said the command is disabled by default. Signed-off-by: Louis Chu --- doctest/build.gradle | 7 ++----- integ-test/build.gradle | 12 ++++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/doctest/build.gradle b/doctest/build.gradle index 1ac658457dc..45f4b768aef 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,11 +205,8 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' - // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into - // the registered endpoints so the rest.md examples run against an enabled command. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it so the rest.md examples run. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c18fa6e37f6..29627ff7b60 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,11 +387,8 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" - // The rest command is disabled by default (empty allow-list). Opt this cluster into the - // registered endpoints so the rest integration tests exercise the enabled path. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it for the rest ITs. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } yamlRestTest { testDistribution = 'archive' @@ -410,9 +407,8 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" - // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' + // Only /_cluster/health is registered; pin the allow-list to it for RestCommandSecurityIT. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } remoteIntegTestWithSecurity { testDistribution = 'archive' From 6a4e66b7898657a64fbf089896566a8f0ab5ac00 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 18:10:33 +0800 Subject: [PATCH 22/23] rest: add a ppl-rest-spi README for endpoint contributors Signed-off-by: Louis Chu --- ppl-rest-spi/README.md | 80 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ppl-rest-spi/README.md diff --git a/ppl-rest-spi/README.md b/ppl-rest-spi/README.md new file mode 100644 index 00000000000..f51dab19f78 --- /dev/null +++ b/ppl-rest-spi/README.md @@ -0,0 +1,80 @@ +# ppl-rest-spi + +The extension point for the PPL `rest` command. Any OpenSearch plugin can contribute read-only, +fixed-schema management endpoints that become queryable as `rest ''`, without touching the +PPL grammar and without depending on the sql plugin's internals. The built-in `/_cluster/health` +endpoint is just one client of this same contract. + +This module is intentionally thin: it depends only on OpenSearch core, and the row values it +exchanges are plain `java.lang` types (String / Number / Boolean / nested Map), so there is no +cross-classloader type-identity problem. + +## Contract + +| Type | Role | +|---|---| +| `RestEndpointProvider` | Your entry point: `List getEndpoints()`. | +| `RestEndpointDefinition` | One endpoint: `name()`, `schema()`, `argSpec()`, `handler()`. Build with `RestEndpointDefinition.builder()`. | +| `Column` / `ColumnType` | A fixed output column. `ColumnType` is `STRING`, `INTEGER`, `LONG`, `DOUBLE`, or `BOOLEAN`. The schema is pinned before execution. | +| `ArgSpec` | The query args the endpoint accepts and each arg's allowed value domain; an unknown or out-of-domain arg is rejected. `ArgSpec.NONE` accepts none. | +| `RestEndpointHandler` | `List> fetch(RestEndpointContext)`, run at scan execution (not planning), so the scan is lazy and `EXPLAIN` is side-effect free. | +| `RestEndpointContext` | The validated `args()` plus an optional core `NodeClient` (`client()`) for a handler that issues its own read-only transport action. | + +## Add an endpoint + +1. Depend on this module `compileOnly`, and declare the sql plugin as an extended plugin so + `loadExtensions` discovers your provider: + + ```gradle + dependencies { compileOnly project(':ppl-rest-spi') } // or the published artifact + opensearchplugin { extendedPlugins = ['opensearch-sql'] } + ``` + +2. Implement `RestEndpointProvider`: + + ```java + public final class MyRestProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_my/thing") + .schema(List.of( + Column.of("name", ColumnType.STRING), + Column.of("count", ColumnType.LONG))) + .argSpec(ArgSpec.builder().arg("verbose", Set.of("true", "false")).build()) + .handler(MyRestProvider::fetch) + .build()); + } + + private static List> fetch(RestEndpointContext ctx) { + // ctx.args() is already validated; ctx.client() is a NodeClient for transport calls (may be null). + return List.of(Map.of("name", "example", "count", 1L)); + } + } + ``` + +3. Register the provider as a service in + `META-INF/services/org.opensearch.sql.spi.rest.RestEndpointProvider`, containing your class's + fully-qualified name. + +4. Enable the endpoint on the cluster. `plugins.ppl.rest.allowed_endpoints` is a node-level setting + listing the endpoint names a deployment permits; it defaults to `["/_cluster/health"]`. Add your + name to run it: + + ```yaml + plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_my/thing"] + ``` + +Then `rest '/_my/thing' verbose=true | where count > 0 | stats sum(count)` composes like any scan. + +## Rules and guarantees + +- Endpoints are read-only. The handler produces rows; the sql side coerces each value to the + declared `ColumnType` and rejects an uncoercible value with a clear error. +- Endpoint names are global. If two providers register the same name, the first registration wins + and the duplicate is ignored with a logged warning. +- An endpoint must be both registered by a provider and present in `allowed_endpoints`; anything + else is rejected before any transport call. +- Redaction of sensitive values is applied centrally at the row-shaping choke point; a provider + does not implement it. From 0ff4abc203b4a19eed39f6774ff71b16a811a06a Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 28 Jul 2026 22:19:11 +0800 Subject: [PATCH 23/23] rest: enumerate the supported endpoints in the resolve() rejection message Reword the resolve() error messages to "Only the following endpoints are supported: ", enumerating the registered allow-list (currently /_cluster/health) instead of promising a vaguer "read-only in-cluster endpoints" category. The "is not allow-listed" clause is kept so callers (and the IT assertion) still see that phrase. Signed-off-by: Louis Chu --- .../sql/opensearch/storage/rest/RestEndpointRegistry.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index d72011c4966..a37438ae3bd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -116,7 +116,7 @@ public List toRows(RestEndpointContext ctx, Redactor redactor) { public Endpoint resolve(String path) { if (path == null || path.isBlank()) { throw new IllegalArgumentException( - "rest endpoint must be a non-empty path. Supported read-only endpoints: " + "rest endpoint must be a non-empty path. Only the following endpoints are supported: " + registry.keySet()); } Endpoint endpoint = registry.get(path); @@ -124,7 +124,7 @@ public Endpoint resolve(String path) { throw new IllegalArgumentException( "rest endpoint [" + path - + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + "] is not allow-listed. Only the following endpoints are supported: " + registry.keySet()); } return endpoint;