diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json index aa36f8fd809aa..41aa3ad5d4113 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json @@ -78,6 +78,10 @@ ], "description":"A health status (\"green\", \"yellow\", or \"red\" to filter only indices matching the specified health status" }, + "system":{ + "type":"boolean", + "description":"If specified, return only system indices when true or only non-system indices when false" + }, "help":{ "type":"boolean", "description":"Return help information", diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 5d16c113339e7..b99896fb95367 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -582,6 +582,7 @@ public class ActionModule extends AbstractModule { private final RequestValidators indicesAliasesRequestRequestValidators; private final ThreadPool threadPool; private final ExtensionsManager extensionsManager; + private final SystemIndices systemIndices; private final ResponseLimitSettings responseLimitSettings; public ActionModule( @@ -607,6 +608,7 @@ public ActionModule( this.actionPlugins = actionPlugins; this.threadPool = threadPool; this.extensionsManager = extensionsManager; + this.systemIndices = systemIndices; actions = setupActions(actionPlugins); actionFilters = setupActionFilters(actionPlugins); dynamicActionRegistry = new DynamicActionRegistry(); @@ -1059,7 +1061,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); registerHandler.accept(new RestTasksAction(nodesInCluster)); - registerHandler.accept(new RestIndicesAction(responseLimitSettings)); + registerHandler.accept(new RestIndicesAction(responseLimitSettings, systemIndices)); registerHandler.accept(new RestSegmentsAction(responseLimitSettings)); // Fully qualified to prevent interference with rest.action.count.RestCountAction registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction()); @@ -1077,7 +1079,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestTemplatesAction()); // LIST API - registerHandler.accept(new RestIndicesListAction(responseLimitSettings)); + registerHandler.accept(new RestIndicesListAction(responseLimitSettings, systemIndices)); registerHandler.accept(new RestShardsListAction()); // Point in time API diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 215ab36a79cdc..94ae806867c38 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -63,6 +63,8 @@ import org.opensearch.core.common.Strings; import org.opensearch.index.IndexSettings; import org.opensearch.index.merge.MergedSegmentWarmerStats; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; @@ -106,10 +108,12 @@ public class RestIndicesAction extends AbstractListAction { private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private final SystemIndices systemIndices; private final ResponseLimitSettings responseLimitSettings; - public RestIndicesAction(ResponseLimitSettings responseLimitSettings) { + public RestIndicesAction(ResponseLimitSettings responseLimitSettings, SystemIndices systemIndices) { this.responseLimitSettings = responseLimitSettings; + this.systemIndices = systemIndices; } @Override @@ -397,7 +401,7 @@ public void onFailure(final Exception e) { private static final Set RESPONSE_PARAMS; static { - final Set responseParams = new HashSet<>(asList("local", "health")); + final Set responseParams = new HashSet<>(asList("local", "health", "system")); responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -430,6 +434,13 @@ protected Table getTableWithHeader(final RestRequest request, final PageToken pa table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); + final String systemColumnDefault = request.hasParam("system") ? "" : "default:false;"; + table.addCell("system", "alias:sys;" + systemColumnDefault + "desc:whether the index is a system index"); + table.addCell( + "system.description", + "alias:sysdesc;" + systemColumnDefault + "desc:description from the matching system index descriptor" + ); + table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); @@ -899,6 +910,7 @@ protected Table buildTable( final PageToken pageToken ) { final String healthParam = request.param("health"); + final Boolean systemParam = request.hasParam("system") ? request.paramAsBoolean("system", false) : null; final Table table = getTableWithHeader(request, pageToken); while (tableIterator.hasNext()) { @@ -942,6 +954,10 @@ protected Table buildTable( } } + if (systemParam != null && systemParam != indexMetadata.isSystem()) { + continue; + } + final CommonStats primaryStats; final CommonStats totalStats; @@ -971,6 +987,10 @@ protected Table buildTable( table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + table.addCell(indexMetadata.isSystem()); + SystemIndexDescriptor descriptor = indexMetadata.isSystem() ? systemIndices.findMatchingDescriptor(indexName) : null; + table.addCell(descriptor == null ? null : descriptor.getDescription()); + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index bd6a9fb09aafe..54d358a66fcc0 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -14,6 +14,7 @@ import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; @@ -36,8 +37,8 @@ public class RestIndicesListAction extends RestIndicesAction { private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; - public RestIndicesListAction(final ResponseLimitSettings responseLimitSettings) { - super(responseLimitSettings); + public RestIndicesListAction(final ResponseLimitSettings responseLimitSettings, final SystemIndices systemIndices) { + super(responseLimitSettings, systemIndices); } @Override diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index 684a1190681c1..0553714ae1987 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -18,6 +18,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.action.admin.cluster.RestCleanupRepositoryAction; import org.opensearch.rest.action.admin.cluster.RestCloneSnapshotAction; @@ -159,7 +160,7 @@ public void testCatIndices() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); Settings settings = Settings.builder().build(); ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); - RestIndicesAction action = new RestIndicesAction(responseLimitSettings); + RestIndicesAction action = new RestIndicesAction(responseLimitSettings, new SystemIndices(Collections.emptyMap())); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index ad3da367477c0..2f8ff69d484fb 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -49,13 +49,19 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.indices.SystemIndices; import org.opensearch.rest.action.list.RestIndicesListAction; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -69,6 +75,9 @@ public class RestIndicesActionTests extends OpenSearchTestCase { + private static final String SYSTEM_INDEX_NAME = ".system-index"; + private static final String SYSTEM_INDEX_DESCRIPTION = "Example of a system index"; + final Map indicesSettings = new LinkedHashMap<>(); final Map indicesMetadatas = new LinkedHashMap<>(); final Map indicesHealths = new LinkedHashMap<>(); @@ -77,9 +86,13 @@ public class RestIndicesActionTests extends OpenSearchTestCase { @Before public void setup() { final int numIndices = randomIntBetween(3, 20); + List indexNames = new ArrayList<>(); + indexNames.add(SYSTEM_INDEX_NAME); for (int i = 0; i < numIndices; i++) { - String indexName = "index-" + i; + indexNames.add("index-" + i); + } + for (String indexName : indexNames) { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) @@ -88,7 +101,7 @@ public void setup() { indicesSettings.put(indexName, indexSettings); IndexMetadata.State indexState = randomBoolean() ? IndexMetadata.State.OPEN : IndexMetadata.State.CLOSE; - if (frequently()) { + if (frequently() || SYSTEM_INDEX_NAME.equals(indexName)) { ClusterHealthStatus healthStatus = randomFrom(ClusterHealthStatus.values()); int numberOfShards = randomIntBetween(1, 3); int numberOfReplicas = healthStatus == ClusterHealthStatus.YELLOW ? 1 : randomInt(1); @@ -98,6 +111,7 @@ public void setup() { .numberOfShards(numberOfShards) .numberOfReplicas(numberOfReplicas) .state(indexState) + .system(SYSTEM_INDEX_NAME.equals(indexName)) .build(); indicesMetadatas.put(indexName, indexMetadata); @@ -146,10 +160,7 @@ public void setup() { } public void testBuildTable() { - final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final Settings settings = Settings.builder().build(); - final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); - final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); + final RestIndicesAction action = newAction(systemIndices()); final Table table = action.buildTable( new FakeRestRequest(), indicesSettings, @@ -160,23 +171,16 @@ public void testBuildTable() { null ); - // now, verify the table is correct assertNotNull(table); - assertTableHeaders(table); - assertThat(table.getRows().size(), equalTo(indicesMetadatas.size())); assertTableRows(table); } public void testBuildPaginatedTable() { - final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final Settings settings = Settings.builder().build(); - final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); - final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); - final RestIndicesListAction indicesListAction = new RestIndicesListAction(responseLimitSettings); + final RestIndicesAction action = newAction(systemIndices()); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(responseLimitSettings(), systemIndices()); List indicesList = new ArrayList<>(indicesMetadatas.keySet()); - // Using half of the indices from metadata list for a page String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); PageToken pageToken = new PageToken("foo", "indices"); final Table table = action.buildTable( @@ -189,18 +193,59 @@ public void testBuildPaginatedTable() { pageToken ); - // verifying table assertNotNull(table); assertTableHeaders(table); assertNotNull(table.getPageToken()); assertEquals(pageToken.getNextToken(), table.getPageToken().getNextToken()); assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity()); - // Table should only contain the indices present in indicesToBeQueried assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() / 2)); assertTableRows(table); } + public void testBuildTableWithSystemParam() { + final RestIndicesAction action = newAction(systemIndices()); + assertTrue(action.responseParams().contains("system")); + final Table table = action.buildTable( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "true")).build(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + action.getTableIterator(new String[0], indicesSettings), + null + ); + + assertTableHeaders(table); + final List> rows = table.getRows(); + assertThat(rows.size(), equalTo(1)); + assertThat(rows.get(0).get(2).value, equalTo(SYSTEM_INDEX_NAME)); + assertThat(rows.get(0).get(headerIndex(table, "system")).value, equalTo(true)); + assertThat(rows.get(0).get(headerIndex(table, "system.description")).value, equalTo(SYSTEM_INDEX_DESCRIPTION)); + } + + public void testBuildTableWithSystemFalseParam() { + final RestIndicesAction action = newAction(systemIndices()); + final Table table = action.buildTable( + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of("system", "false")).build(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + action.getTableIterator(new String[0], indicesSettings), + null + ); + + assertTableHeaders(table); + final int systemColumn = headerIndex(table, "system"); + final int descriptionColumn = headerIndex(table, "system.description"); + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() - 1)); + for (List row : table.getRows()) { + assertThat(row.get(systemColumn).value, equalTo(false)); + assertNull(row.get(descriptionColumn).value); + } + } + private void assertTableHeaders(Table table) { List headers = table.getHeaders(); assertThat(headers.get(0).value, equalTo("health")); @@ -209,8 +254,10 @@ private void assertTableHeaders(Table table) { assertThat(headers.get(3).value, equalTo("uuid")); assertThat(headers.get(4).value, equalTo("pri")); assertThat(headers.get(5).value, equalTo("rep")); - // Check for new columns (at the end) - boolean foundRaw = false, foundString = false; + assertTrue(headerIndex(table, "system") > 5); + assertTrue(headerIndex(table, "system.description") > 5); + boolean foundRaw = false; + boolean foundString = false; for (Table.Cell cell : headers) { if ("last_index_request_timestamp".equals(cell.value)) foundRaw = true; if ("last_index_request_timestamp_string".equals(cell.value)) foundString = true; @@ -250,12 +297,18 @@ private void assertTableRows(Table table) { } } + private int headerIndex(Table table, String name) { + for (int i = 0; i < table.getHeaders().size(); i++) { + if (name.equals(table.getHeaders().get(i).value)) { + return i; + } + } + fail("missing table header [" + name + "]"); + return -1; + } + public void testLastIndexRequestTimestampColumns() { - final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final Settings settings = Settings.builder().build(); - final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); - final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); - // Setup a known timestamp + final RestIndicesAction action = newAction(emptySystemIndices()); long knownTs = 1710000000000L; IndexStats indexStats = mock(IndexStats.class); CommonStats commonStats = mock(CommonStats.class); @@ -272,7 +325,7 @@ public void testLastIndexRequestTimestampColumns() { Map testSettings = new LinkedHashMap<>(); testSettings.put(testIndex, Settings.EMPTY); Map testMetadatas = new LinkedHashMap<>(); - Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build(); + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); testMetadatas.put( testIndex, IndexMetadata.builder(testIndex).settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build() @@ -287,9 +340,10 @@ public void testLastIndexRequestTimestampColumns() { action.getTableIterator(new String[] { testIndex }, testSettings), null ); - // Find the columns + List header = table.getHeaders(); - int rawIdx = -1, strIdx = -1; + int rawIdx = -1; + int strIdx = -1; for (int i = 0; i < header.size(); i++) { if ("last_index_request_timestamp".equals(header.get(i).value)) rawIdx = i; if ("last_index_request_timestamp_string".equals(header.get(i).value)) strIdx = i; @@ -300,13 +354,29 @@ public void testLastIndexRequestTimestampColumns() { assertEquals(1, rows.size()); List row = rows.get(0); assertEquals(String.valueOf(knownTs), row.get(rawIdx).value.toString()); - // Robust: parse the string as ISO-8601 and compare to knownTs String timestampString = row.get(strIdx).value.toString(); try { - java.time.Instant parsed = java.time.Instant.parse(timestampString); + Instant parsed = Instant.parse(timestampString); assertEquals(knownTs, parsed.toEpochMilli()); - } catch (java.time.format.DateTimeParseException e) { + } catch (DateTimeParseException e) { fail("Timestamp string is not a valid ISO-8601 date: " + timestampString); } } + + private RestIndicesAction newAction(SystemIndices systemIndices) { + return new RestIndicesAction(responseLimitSettings(), systemIndices); + } + + private ResponseLimitSettings responseLimitSettings() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + return new ResponseLimitSettings(clusterSettings, Settings.EMPTY); + } + + private SystemIndices systemIndices() { + return new SystemIndices(Map.of("testplugin", List.of(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, SYSTEM_INDEX_DESCRIPTION)))); + } + + private SystemIndices emptySystemIndices() { + return new SystemIndices(Collections.emptyMap()); + } }