From 8b358525793dc27cf1113bfbfd49227467161d0e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 14 Nov 2023 14:53:18 -0500 Subject: [PATCH 1/4] Add system param to _cat/indices to only list system indices from SystemIndexPlugin.getSystemIndexDescriptors Signed-off-by: Craig Perkins --- .../org/opensearch/action/ActionModule.java | 4 +- .../rest/action/cat/RestIndicesAction.java | 16 ++++++ .../RenamedTimeoutRequestParameterTests.java | 3 +- .../action/cat/RestIndicesActionTests.java | 52 ++++++++++++++++--- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 46775466aa615..45087d72044a2 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -512,6 +512,7 @@ public class ActionModule extends AbstractModule { private final RequestValidators indicesAliasesRequestRequestValidators; private final ThreadPool threadPool; private final ExtensionsManager extensionsManager; + private final SystemIndices systemIndices; public ActionModule( Settings settings, @@ -536,6 +537,7 @@ public ActionModule( this.actionPlugins = actionPlugins; this.threadPool = threadPool; this.extensionsManager = extensionsManager; + this.systemIndices = systemIndices; actions = setupActions(actionPlugins); actionFilters = setupActionFilters(actionPlugins); dynamicActionRegistry = new DynamicActionRegistry(); @@ -922,7 +924,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); registerHandler.accept(new RestTasksAction(nodesInCluster)); - registerHandler.accept(new RestIndicesAction()); + registerHandler.accept(new RestIndicesAction(systemIndices)); registerHandler.accept(new RestSegmentsAction()); // Fully qualified to prevent interference with rest.action.count.RestCountAction registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction()); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 23cc1cb507072..217e07bc64b73 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -59,6 +59,7 @@ import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; @@ -97,6 +98,12 @@ public class RestIndicesAction extends AbstractCatAction { private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private final SystemIndices systemIndices; + + public RestIndicesAction(SystemIndices systemIndices) { + this.systemIndices = systemIndices; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); @@ -123,6 +130,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()); final boolean local = request.paramAsBoolean("local", false); + final boolean system = request.paramAsBoolean("system", false); TimeValue clusterManagerTimeout = request.paramAsTime("cluster_manager_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); // Remove the if condition and statements inside after removing MASTER_ROLE. if (request.hasParam("master_timeout")) { @@ -716,6 +724,7 @@ Table buildTable( ) { final String healthParam = request.param("health"); + final boolean systemParam = request.paramAsBoolean("system", false); final Table table = getTableWithHeader(request); indicesSettings.forEach((indexName, settings) -> { @@ -755,6 +764,13 @@ Table buildTable( } } + if (systemParam) { + if (!systemIndices.isSystemIndex(indexName)) { + // System filter is enabled but index is not a system index + return; + } + } + final CommonStats primaryStats; final CommonStats totalStats; diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index c7e1039686cc9..43843aef0d483 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -17,6 +17,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.action.admin.cluster.RestCleanupRepositoryAction; import org.opensearch.rest.action.admin.cluster.RestCloneSnapshotAction; @@ -155,7 +156,7 @@ public void testCatAllocation() { } public void testCatIndices() { - RestIndicesAction action = new RestIndicesAction(); + RestIndicesAction action = new RestIndicesAction(new SystemIndices(Collections.emptyMap())); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 96b1c75371697..adb4073c4fff4 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -46,10 +46,14 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.indices.SystemIndices; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -70,9 +74,15 @@ public void testBuildTable() { final Map indicesHealths = new LinkedHashMap<>(); final Map indicesStats = new LinkedHashMap<>(); + String systemIndexName = ".system-index"; + List indexNames = new ArrayList<>(); + indexNames.add(systemIndexName); for (int i = 0; i < numIndices; i++) { String indexName = "index-" + i; + indexNames.add(indexName); + } + for (String indexName : indexNames) { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) @@ -137,17 +147,13 @@ public void testBuildTable() { } } - final RestIndicesAction action = new RestIndicesAction(); + final RestIndicesAction action = new RestIndicesAction( + new SystemIndices(Map.of("testplugin", List.of(new SystemIndexDescriptor(systemIndexName, "Example of a system index")))) + ); final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas); // now, verify the table is correct - List headers = table.getHeaders(); - assertThat(headers.get(0).value, equalTo("health")); - assertThat(headers.get(1).value, equalTo("status")); - assertThat(headers.get(2).value, equalTo("index")); - assertThat(headers.get(3).value, equalTo("uuid")); - assertThat(headers.get(4).value, equalTo("pri")); - assertThat(headers.get(5).value, equalTo("rep")); + verifyTableHeaders(table); final List> rows = table.getRows(); assertThat(rows.size(), equalTo(indicesMetadatas.size())); @@ -178,5 +184,35 @@ public void testBuildTable() { assertThat(row.get(5).value, nullValue()); } } + + final Table tableWithSystemIndexOnly = action.buildTable( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "true")).build(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas + ); + + // now, verify the table is correct + verifyTableHeaders(tableWithSystemIndexOnly); + + final List> systemIndexRows = tableWithSystemIndexOnly.getRows(); + assertThat(systemIndexRows.size(), equalTo(1)); + + for (final List row : systemIndexRows) { + final String indexName = (String) row.get(2).value; + + assertThat(indexName, equalTo(".system-index")); + } + } + + private void verifyTableHeaders(Table table) { + List headers = table.getHeaders(); + assertThat(headers.get(0).value, equalTo("health")); + assertThat(headers.get(1).value, equalTo("status")); + assertThat(headers.get(2).value, equalTo("index")); + assertThat(headers.get(3).value, equalTo("uuid")); + assertThat(headers.get(4).value, equalTo("pri")); + assertThat(headers.get(5).value, equalTo("rep")); } } From 4af2dd558f3195b7a153a77efd2e5247733ff7f1 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 14 Nov 2023 14:59:57 -0500 Subject: [PATCH 2/4] Add entry to CHANGELOG Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index daf5568fa7a77..a38e4a112354e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add cluster state stats ([#10670](https://github.com/opensearch-project/OpenSearch/pull/10670)) - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069)) - [Streaming Indexing] Introduce new experimental server HTTP transport based on Netty 4 and Project Reactor (Reactor Netty) ([#9672](https://github.com/opensearch-project/OpenSearch/pull/9672)) +- Add system param to /_cat/indices to only list system indices from SystemIndexPlugin.getSystemIndexDescriptors ([11201](https://github.com/opensearch-project/OpenSearch/pull/11201)) ### Dependencies - Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.1 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208), [#10298](https://github.com/opensearch-project/OpenSearch/pull/10298)) From 7ad8bc7d54eed5e01174ee857c297be79e6fdcff Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 15 Nov 2023 12:03:10 -0500 Subject: [PATCH 3/4] Add column for description as well Signed-off-by: Craig Perkins --- .../opensearch/rest/action/cat/RestIndicesAction.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 217e07bc64b73..57f1957b78b0e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -59,6 +59,7 @@ import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; @@ -366,6 +367,11 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); + final boolean systemParam = request.paramAsBoolean("system", false); + if (systemParam) { + table.addCell("desc", "alias:dsc;desc:description of system index"); + } + table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); @@ -800,6 +806,11 @@ Table buildTable( table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + if (systemParam) { + SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); + table.addCell(descriptor == null ? null : descriptor.getDescription()); + } + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); From 27195f649f1107b593c6d877783176e48b81d2f3 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sun, 12 Jul 2026 23:15:14 -0400 Subject: [PATCH 4/4] Expose system index metadata in cat indices Signed-off-by: Craig Perkins --- .../rest-api-spec/api/cat.indices.json | 4 ++ .../rest/action/cat/RestIndicesAction.java | 29 ++++++------- .../action/cat/RestIndicesActionTests.java | 42 +++++++++++++++++-- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json index aa36f8fd809aa..41aa3ad5d4113 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json @@ -78,6 +78,10 @@ ], "description":"A health status (\"green\", \"yellow\", or \"red\" to filter only indices matching the specified health status" }, + "system":{ + "type":"boolean", + "description":"If specified, return only system indices when true or only non-system indices when false" + }, "help":{ "type":"boolean", "description":"Return help information", diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index e35ea1dd252c9..94ae806867c38 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -147,7 +147,6 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()); final boolean local = request.paramAsBoolean("local", false); - final boolean system = request.paramAsBoolean("system", false); TimeValue clusterManagerTimeout = request.paramAsTime("cluster_manager_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); // Remove the if condition and statements inside after removing MASTER_ROLE. if (request.hasParam("master_timeout")) { @@ -402,7 +401,7 @@ public void onFailure(final Exception e) { private static final Set RESPONSE_PARAMS; static { - final Set responseParams = new HashSet<>(asList("local", "health")); + final Set responseParams = new HashSet<>(asList("local", "health", "system")); responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -435,10 +434,12 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); - final boolean systemParam = request.paramAsBoolean("system", false); - if (systemParam) { - table.addCell("desc", "alias:dsc;desc:description of system index"); - } + final String systemColumnDefault = request.hasParam("system") ? "" : "default:false;"; + table.addCell("system", "alias:sys;" + systemColumnDefault + "desc:whether the index is a system index"); + table.addCell( + "system.description", + "alias:sysdesc;" + systemColumnDefault + "desc:description from the matching system index descriptor" + ); table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); @@ -909,7 +910,7 @@ protected Table buildTable( final PageToken pageToken ) { final String healthParam = request.param("health"); - final boolean systemParam = request.paramAsBoolean("system", false); + final Boolean systemParam = request.hasParam("system") ? request.paramAsBoolean("system", false) : null; final Table table = getTableWithHeader(request, pageToken); while (tableIterator.hasNext()) { @@ -953,11 +954,8 @@ protected Table buildTable( } } - if (systemParam) { - if (!systemIndices.isSystemIndex(indexName)) { - // System filter is enabled but index is not a system index - continue; - } + if (systemParam != null && systemParam != indexMetadata.isSystem()) { + continue; } final CommonStats primaryStats; @@ -989,10 +987,9 @@ protected Table buildTable( table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); - if (systemParam) { - SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName); - table.addCell(descriptor == null ? null : descriptor.getDescription()); - } + table.addCell(indexMetadata.isSystem()); + SystemIndexDescriptor descriptor = indexMetadata.isSystem() ? systemIndices.findMatchingDescriptor(indexName) : null; + table.addCell(descriptor == null ? null : descriptor.getDescription()); table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index c8884ca95c554..2f8ff69d484fb 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -111,6 +111,7 @@ public void setup() { .numberOfShards(numberOfShards) .numberOfReplicas(numberOfReplicas) .state(indexState) + .system(SYSTEM_INDEX_NAME.equals(indexName)) .build(); indicesMetadatas.put(indexName, indexMetadata); @@ -204,6 +205,7 @@ public void testBuildPaginatedTable() { public void testBuildTableWithSystemParam() { final RestIndicesAction action = newAction(systemIndices()); + assertTrue(action.responseParams().contains("system")); final Table table = action.buildTable( new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "true")).build(), indicesSettings, @@ -215,13 +217,33 @@ public void testBuildTableWithSystemParam() { ); assertTableHeaders(table); - List headers = table.getHeaders(); - assertThat(headers.get(12).value, equalTo("desc")); - final List> rows = table.getRows(); assertThat(rows.size(), equalTo(1)); assertThat(rows.get(0).get(2).value, equalTo(SYSTEM_INDEX_NAME)); - assertThat(rows.get(0).get(12).value, equalTo(SYSTEM_INDEX_DESCRIPTION)); + assertThat(rows.get(0).get(headerIndex(table, "system")).value, equalTo(true)); + assertThat(rows.get(0).get(headerIndex(table, "system.description")).value, equalTo(SYSTEM_INDEX_DESCRIPTION)); + } + + public void testBuildTableWithSystemFalseParam() { + final RestIndicesAction action = newAction(systemIndices()); + final Table table = action.buildTable( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "false")).build(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + action.getTableIterator(new String[0], indicesSettings), + null + ); + + assertTableHeaders(table); + final int systemColumn = headerIndex(table, "system"); + final int descriptionColumn = headerIndex(table, "system.description"); + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() - 1)); + for (List row : table.getRows()) { + assertThat(row.get(systemColumn).value, equalTo(false)); + assertNull(row.get(descriptionColumn).value); + } } private void assertTableHeaders(Table table) { @@ -232,6 +254,8 @@ private void assertTableHeaders(Table table) { assertThat(headers.get(3).value, equalTo("uuid")); assertThat(headers.get(4).value, equalTo("pri")); assertThat(headers.get(5).value, equalTo("rep")); + assertTrue(headerIndex(table, "system") > 5); + assertTrue(headerIndex(table, "system.description") > 5); boolean foundRaw = false; boolean foundString = false; for (Table.Cell cell : headers) { @@ -273,6 +297,16 @@ private void assertTableRows(Table table) { } } + private int headerIndex(Table table, String name) { + for (int i = 0; i < table.getHeaders().size(); i++) { + if (name.equals(table.getHeaders().get(i).value)) { + return i; + } + } + fail("missing table header [" + name + "]"); + return -1; + } + public void testLastIndexRequestTimestampColumns() { final RestIndicesAction action = newAction(emptySystemIndices()); long knownTs = 1710000000000L;