diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java index 1fa5dec3da471..188c0dbda00fb 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AggregateFunction.java @@ -40,6 +40,8 @@ public enum AggregateFunction { PERCENTILE_CONT(Type.STATE_EXPANDING, SqlKind.PERCENTILE_CONT), PERCENTILE_DISC(Type.STATE_EXPANDING, SqlKind.PERCENTILE_DISC), PERCENTILE_APPROX(Type.STATE_EXPANDING, SqlKind.OTHER), + /** Explicit-compression percentile: {@code (field, percent, centroids)}. */ + PERCENTILE_APPROX_N(Type.STATE_EXPANDING, SqlKind.OTHER), COLLECT(Type.STATE_EXPANDING, SqlKind.COLLECT), LISTAGG(Type.STATE_EXPANDING, SqlKind.LISTAGG), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index 6831e27bb06e2..ff60d932f2892 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -475,6 +475,7 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP AggregateFunction.AVG, AggregateFunction.APPROX_COUNT_DISTINCT, AggregateFunction.PERCENTILE_APPROX, + AggregateFunction.PERCENTILE_APPROX_N, AggregateFunction.TAKE, AggregateFunction.FIRST, AggregateFunction.LAST, diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 4e265ff0d09b6..c0d0e6f677237 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -38,6 +38,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.util.ImmutableBitSet; @@ -381,6 +382,23 @@ public RexNode normaliseLiteralArg(int argIndex, RexLiteral lit, RexBuilder rexB } }; + /** + * Explicit-compression percentile (DSL executor): {@code PERCENTILE_APPROX_N(field, + * percent, centroids)} → {@code approx_percentile_cont}. Percent rescaling delegates to + * {@link #LOCAL_PERCENTILE_APPROX_OP}; the centroids literal passes through unrescaled. + */ + static final LocalAggOp LOCAL_PERCENTILE_APPROX_N_OP = new LocalAggOp( + "approx_percentile_cont", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0.andThen(SqlTypeTransforms.FORCE_NULLABLE), + OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.ANY, SqlTypeFamily.ANY) + ) { + @Override + public RexNode normaliseLiteralArg(int argIndex, RexLiteral lit, RexBuilder rexBuilder, RelDataTypeFactory typeFactory) { + return LOCAL_PERCENTILE_APPROX_OP.normaliseLiteralArg(argIndex, lit, rexBuilder, typeFactory); + } + }; + /** BRAIN window stub for {@code patterns ... method=BRAIN mode=label}. */ static final SqlAggFunction LOCAL_INTERNAL_PATTERN_WINDOW_OP = new SqlAggFunction( "internal_pattern", @@ -440,6 +458,7 @@ public RexNode normaliseLiteralArg(int argIndex, RexLiteral lit, RexBuilder rexB FunctionMappings.s(LOCAL_LIST_MERGE_OP, "list_merge"), FunctionMappings.s(LOCAL_LIST_MERGE_DISTINCT_OP, "list_merge_distinct"), FunctionMappings.s(LOCAL_PERCENTILE_APPROX_OP, "approx_percentile_cont"), + FunctionMappings.s(LOCAL_PERCENTILE_APPROX_N_OP, "approx_percentile_cont"), FunctionMappings.s(LOCAL_INTERNAL_PATTERN_OP, "internal_pattern"), FunctionMappings.s(LOCAL_OS_COUNT_DISTINCT_OP, "os_count_distinct") ); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PplAggregateCallRewriter.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PplAggregateCallRewriter.java index bc71c033508b3..ca3821e1fba2c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PplAggregateCallRewriter.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/PplAggregateCallRewriter.java @@ -30,8 +30,9 @@ import java.util.Set; /** - * Rewrites PPL state-expanding aggregates (TAKE / FIRST / LAST / LIST / VALUES / PATTERN / - * PERCENTILE_APPROX) onto local stubs the substrait emitter binds via + * Rewrites state-expanding aggregates (TAKE / FIRST / LAST / LIST / VALUES / PATTERN / + * PERCENTILE_APPROX / PERCENTILE_APPROX_N — emitted by PPL and, for the percentile ops, + * by the DSL query executor) onto local stubs the substrait emitter binds via * {@link DataFusionFragmentConvertor}'s ADDITIONAL_AGGREGATE_SIGS. Also normalises any * RexLiteral{SqlTypeName.SYMBOL} in upstream Projects to VARCHAR — isthmus's * LiteralConverter rejects unregistered Enum classes, and PPL's percentile_approx / @@ -186,14 +187,18 @@ private static AggregateCall rewriteCall(Aggregate agg, AggregateCall call) { targetOp = DataFusionFragmentConvertor.LOCAL_INTERNAL_PATTERN_OP; explicitReturnType = internalPatternReturnType(agg.getCluster().getTypeFactory()); } - case "PERCENTILE_APPROX" -> { - // Trim the PPL type-flag arg; the substrait emit-time literal-arg normaliser - // (DataFusionFragmentConvertor#normaliseLiteralArg) rescales the percentile. - if (call.getArgList().size() < 3) { + case "PERCENTILE_APPROX", "PERCENTILE_APPROX_N" -> { + // PERCENTILE_APPROX: (field, percent) + optional PPL type-flag arg to trim. + // PERCENTILE_APPROX_N (DSL executor): (field, percent, centroids), kept whole. + boolean withCentroids = "PERCENTILE_APPROX_N".equals(call.getAggregation().getName()); + int argCount = withCentroids ? 3 : 2; + if (call.getArgList().size() < argCount) { return call; } - targetOp = DataFusionFragmentConvertor.LOCAL_PERCENTILE_APPROX_OP; - List trimmedArgList = new ArrayList<>(call.getArgList().subList(0, 2)); + targetOp = withCentroids + ? DataFusionFragmentConvertor.LOCAL_PERCENTILE_APPROX_N_OP + : DataFusionFragmentConvertor.LOCAL_PERCENTILE_APPROX_OP; + List trimmedArgList = new ArrayList<>(call.getArgList().subList(0, argCount)); RelDataType arg0Type = agg.getInput().getRowType().getFieldList().get(call.getArgList().get(0)).getType(); RelDataType nullableArg0 = agg.getCluster().getTypeFactory().createTypeWithNullability(arg0Type, true); return AggregateCall.create( diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml index c0ec1000a4337..3049680f092f8 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/resources/opensearch_aggregate_functions.yaml @@ -109,7 +109,9 @@ aggregate_functions: - name: approx_percentile_cont description: >- PPL `percentile_approx(field, percentile)` — t-digest based approximate - percentile. Maps to DataFusion's builtin `approx_percentile_cont`. + percentile. Maps to DataFusion's builtin `approx_percentile_cont`. The + three-arg overload carries an explicit centroid count (DSL percentiles + `tdigest.compression`, PERCENTILE_APPROX_N). impls: - args: - name: x @@ -117,6 +119,14 @@ aggregate_functions: - name: percentile value: any2 return: any1 + - args: + - name: x + value: any1 + - name: percentile + value: any2 + - name: centroids + value: any3 + return: any1 - name: os_count_distinct description: >- Exact distinct count for use in a window context. PPL diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggLiteralArgProjectSplitRule.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggLiteralArgProjectSplitRule.java index 79ee86d167816..ca8792e9a0686 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggLiteralArgProjectSplitRule.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/planner/rules/OpenSearchAggLiteralArgProjectSplitRule.java @@ -44,17 +44,17 @@ * in the coordinator fragment; the unpinned lower copy narrows the scan. {@code argList} is untouched. * *

Runs after marking, before CBO — {@code PROJECT_MERGE} (pre-marking) can't re-fuse the copies, and the - * ER lands between them. Scope = {@code PERCENTILE_APPROX} and {@code TAKE}, the only aggregates that - * materialize a literal config arg as a Project column (FIRST/LAST drop their optional N; others carry no - * literal). Fires only when such a call references a literal column and the Project also has a non-literal - * column to push down. + * ER lands between them. Scope = {@code PERCENTILE_APPROX} / {@code PERCENTILE_APPROX_N} and {@code TAKE}, + * the only aggregates that materialize literal config args as Project columns (FIRST/LAST drop their + * optional N; others carry no literal). Fires only when such a call references a literal column and the + * Project also has a non-literal column to push down. * * @opensearch.internal */ public class OpenSearchAggLiteralArgProjectSplitRule extends RelOptRule { - // Aggregates whose literal config arg the DataFusion converter re-inlines from the attached Project. - private static final java.util.Set LITERAL_ARG_AGGS = java.util.Set.of("PERCENTILE_APPROX", "TAKE"); + // Aggregates whose literal config args the DataFusion converter re-inlines from the attached Project. + private static final java.util.Set LITERAL_ARG_AGGS = java.util.Set.of("PERCENTILE_APPROX", "PERCENTILE_APPROX_N", "TAKE"); public OpenSearchAggLiteralArgProjectSplitRule() { super(operand(OpenSearchAggregate.class, operand(OpenSearchProject.class, none())), "OpenSearchAggLiteralArgProjectSplitRule"); diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java index b9ec426791835..7313bf4c04310 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/action/TransportDslExecuteAction.java @@ -78,14 +78,13 @@ public TransportDslExecuteAction( @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { threadPool.executor(ThreadPool.Names.SEARCH).execute(() -> { + final long startTime = System.currentTimeMillis(); final QueryPlans plans; - final long convertTime; + final SearchSourceConverter converter; try { String indexName = resolveToSingleIndex(request); - long convertStart = System.nanoTime(); - SearchSourceConverter converter = new SearchSourceConverter(contextProvider.getContext().schema()); + converter = new SearchSourceConverter(contextProvider.getContext().schema()); plans = converter.convert(request.source(), indexName); - convertTime = System.nanoTime() - convertStart; } catch (Exception e) { logger.error("DSL conversion failed", e); listener.onFailure(e); @@ -94,7 +93,8 @@ protected void doExecute(Task task, SearchRequest request, ActionListener { final SearchResponse response; try { - response = SearchResponseBuilder.build(results, convertTime); + long tookInMillis = System.currentTimeMillis() - startTime; + response = SearchResponseBuilder.build(results, request, converter.getAggregationRegistry(), tookInMillis); } catch (Exception buildEx) { logger.error("DSL response building failed", buildEx); listener.onFailure(buildEx); diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadata.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadata.java index 783aa34ab71e9..7916800f6546c 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadata.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadata.java @@ -31,6 +31,7 @@ public class AggregationMetadata { private final List aggregateCalls; private final List aggregateFieldNames; private final List bucketOrders; + private final List literalColumns; /** * Creates aggregation metadata. @@ -40,19 +41,23 @@ public class AggregationMetadata { * @param aggregateCalls Calcite aggregate calls (AVG, SUM, etc.) * @param aggregateFieldNames output names for aggregate results * @param bucketOrders bucket orders for post-aggregation sorting + * @param literalColumns literal-derived columns to append to the aggregate's input, in + * allocation order (see {@link LiteralColumnAllocator}) */ public AggregationMetadata( ImmutableBitSet groupByBitSet, List groupByFieldNames, List aggregateCalls, List aggregateFieldNames, - List bucketOrders + List bucketOrders, + List literalColumns ) { this.groupByBitSet = groupByBitSet; this.groupByFieldNames = List.copyOf(groupByFieldNames); this.aggregateCalls = List.copyOf(aggregateCalls); this.aggregateFieldNames = List.copyOf(aggregateFieldNames); this.bucketOrders = List.copyOf(bucketOrders); + this.literalColumns = List.copyOf(literalColumns); } /** Returns the GROUP BY column indices. */ @@ -84,4 +89,9 @@ public List getBucketOrders() { public boolean hasBucketOrders() { return !bucketOrders.isEmpty(); } + + /** Returns the literal-derived columns to append to the aggregate's input, in allocation order. */ + public List getLiteralColumns() { + return literalColumns; + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilder.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilder.java index 72ae645beb534..b798ee49a45cd 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilder.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilder.java @@ -13,6 +13,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.ImmutableBitSet; @@ -37,6 +38,7 @@ public class AggregationMetadataBuilder { private final List aggregateCalls = new ArrayList<>(); private final List aggregateFieldNames = new ArrayList<>(); private final List bucketOrders = new ArrayList<>(); + private final List literalColumns = new ArrayList<>(); private boolean implicitCountRequested = false; /** Creates a new empty builder. */ @@ -85,6 +87,41 @@ public void requestImplicitCount() { this.implicitCountRequested = true; } + /** + * Returns a literal-column allocator. Allocated columns are appended after the + * {@code baseFieldCount} input fields (deduplicated) and materialized by the converter + * in a pre-aggregate project. + * + * @param baseFieldCount the field count of the aggregate's un-projected input + */ + public LiteralColumnAllocator literalColumnAllocator(int baseFieldCount) { + return new LiteralColumnAllocator() { + @Override + public int columnFor(double value) { + return indexFor(LiteralColumn.constant(value)); + } + + @Override + public int integerColumnFor(long value) { + return indexFor(LiteralColumn.integerConstant(value)); + } + + @Override + public int coalescedColumnFor(int fieldIndex, double missingValue) { + return indexFor(LiteralColumn.coalesced(fieldIndex, missingValue)); + } + + private int indexFor(LiteralColumn column) { + int existing = literalColumns.indexOf(column); + if (existing >= 0) { + return baseFieldCount + existing; + } + literalColumns.add(column); + return baseFieldCount + literalColumns.size() - 1; + } + }; + } + /** Returns true if this builder has at least one aggregate call or implicit count. */ public boolean hasAggregateCalls() { return !aggregateCalls.isEmpty() || implicitCountRequested; @@ -113,8 +150,24 @@ public AggregationMetadata build(RelDataType inputRowType, RelDataTypeFactory ty boolean noGroupBy = groupings.isEmpty(); List allCalls = new ArrayList<>(); for (AggregateCall call : aggregateCalls) { - if (noGroupBy) { - RelDataType nullableType = typeFactory.createTypeWithNullability(call.getType(), true); + boolean isCount = call.getAggregation().getKind() == SqlKind.COUNT; + String functionName = call.getAggregation().getName(); + // Percentile results are nullable even under GROUP BY (engine may emit NULL per group). + boolean forcedNullable = "PERCENTILE_APPROX".equals(functionName) || "PERCENTILE_APPROX_N".equals(functionName); + RelDataType targetType; + if (isCount) { + // COUNT is always BIGINT NOT NULL regardless of the input field's type; normalize here + // (translators have no type factory) — LogicalAggregate.create asserts the type matches. + targetType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), false); + } else if (noGroupBy || forcedNullable) { + // AVG, MIN, MAX, SUM return null for empty sets when there's no GROUP BY. + targetType = typeFactory.createTypeWithNullability(call.getType(), true); + } else { + targetType = call.getType(); + } + if (targetType.equals(call.getType())) { + allCalls.add(call); + } else { allCalls.add( AggregateCall.create( call.getAggregation(), @@ -124,12 +177,10 @@ public AggregationMetadata build(RelDataType inputRowType, RelDataTypeFactory ty call.getArgList(), call.filterArg, call.getCollation(), - nullableType, + targetType, call.getName() ) ); - } else { - allCalls.add(call); } } List allFieldNames = new ArrayList<>(aggregateFieldNames); @@ -144,14 +195,21 @@ public AggregationMetadata build(RelDataType inputRowType, RelDataTypeFactory ty List.of(), -1, RelCollations.EMPTY, - typeFactory.createSqlType(SqlTypeName.BIGINT), + typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), false), IMPLICIT_COUNT_NAME ) ); allFieldNames.add(IMPLICIT_COUNT_NAME); } - return new AggregationMetadata(ImmutableBitSet.of(allGroupIndices), allGroupFieldNames, allCalls, allFieldNames, bucketOrders); + return new AggregationMetadata( + ImmutableBitSet.of(allGroupIndices), + allGroupFieldNames, + allCalls, + allFieldNames, + bucketOrders, + literalColumns + ); } /** diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationRegistryFactory.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationRegistryFactory.java index 9297f6ca5cefd..6796338f05365 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationRegistryFactory.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationRegistryFactory.java @@ -10,9 +10,13 @@ import org.opensearch.dsl.aggregation.bucket.TermsBucketTranslator; import org.opensearch.dsl.aggregation.metric.AvgMetricTranslator; +import org.opensearch.dsl.aggregation.metric.ExtendedStatsMetricTranslator; import org.opensearch.dsl.aggregation.metric.MaxMetricTranslator; import org.opensearch.dsl.aggregation.metric.MinMetricTranslator; +import org.opensearch.dsl.aggregation.metric.PercentilesMetricTranslator; +import org.opensearch.dsl.aggregation.metric.StatsMetricTranslator; import org.opensearch.dsl.aggregation.metric.SumMetricTranslator; +import org.opensearch.dsl.aggregation.metric.ValueCountMetricTranslator; /** * Creates an {@link AggregationRegistry} populated with all supported translators. @@ -28,8 +32,11 @@ public static AggregationRegistry create() { registry.register(new SumMetricTranslator()); registry.register(new MinMetricTranslator()); registry.register(new MaxMetricTranslator()); + registry.register(new StatsMetricTranslator()); + registry.register(new ExtendedStatsMetricTranslator()); + registry.register(new ValueCountMetricTranslator()); + registry.register(new PercentilesMetricTranslator()); registry.register(new TermsBucketTranslator()); - // TODO: add other aggregation translators return registry; } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationTreeWalker.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationTreeWalker.java index 5f0c0600e1bcf..039bc90a35aba 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationTreeWalker.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/AggregationTreeWalker.java @@ -8,6 +8,7 @@ package org.opensearch.dsl.aggregation; +import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.opensearch.dsl.aggregation.bucket.BucketTranslator; @@ -120,7 +121,25 @@ private void handleMetric( RelDataType rowType ) throws ConversionException { AggregationMetadataBuilder builder = getOrCreateBuilder(currentGroupings, granularities); - builder.addAggregateCall(translator.toAggregateCall(aggBuilder, rowType), translator.getAggregateFieldName(aggBuilder)); + List calls = translator.toAggregateCalls( + aggBuilder, + rowType, + builder.literalColumnAllocator(rowType.getFieldCount()) + ); + List fieldNames = translator.getAggregateFieldNames(aggBuilder); + if (calls.size() != fieldNames.size()) { + throw new ConversionException( + "Mismatch between aggregate calls (" + + calls.size() + + ") and field names (" + + fieldNames.size() + + ") for aggregation: " + + aggBuilder.getName() + ); + } + for (int i = 0; i < calls.size(); i++) { + builder.addAggregateCall(calls.get(i), fieldNames.get(i)); + } } private AggregationMetadataBuilder getOrCreateBuilder( diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumn.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumn.java new file mode 100644 index 0000000000000..628bba220cb63 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumn.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation; + +/** + * One literal-derived input column of the aggregate, materialized by the converter in the + * pre-aggregate project. Three kinds: + * + *

+ * + *

Record equality drives allocator dedup: equal columns share one projected column. + * + * @param kind which of the three column kinds this is + * @param coalesceFieldIndex input index of the coalesced field, or {@code null} for constants + * @param value the constant value, or the {@code missing} substitute for a coalesced column + */ +public record LiteralColumn(Kind kind, Integer coalesceFieldIndex, double value) { + + /** The three column kinds. */ + public enum Kind { + /** Constant typed as DOUBLE. */ + DOUBLE_CONSTANT, + /** Constant typed as an exact integer. */ + INTEGER_CONSTANT, + /** {@code COALESCE(field, value)}. */ + COALESCED + } + + /** + * Creates a DOUBLE constant column. + * + * @param value the constant value + */ + public static LiteralColumn constant(double value) { + return new LiteralColumn(Kind.DOUBLE_CONSTANT, null, value); + } + + /** + * Creates an exact-integer constant column. + * + * @param value the constant value + */ + public static LiteralColumn integerConstant(long value) { + return new LiteralColumn(Kind.INTEGER_CONSTANT, null, value); + } + + /** + * Creates a coalesced column: {@code COALESCE(field, missingValue)}. + * + * @param fieldIndex input index of the field + * @param missingValue substitute for SQL NULL + */ + public static LiteralColumn coalesced(int fieldIndex, double missingValue) { + return new LiteralColumn(Kind.COALESCED, fieldIndex, missingValue); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumnAllocator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumnAllocator.java new file mode 100644 index 0000000000000..80d82cb346ffd --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/LiteralColumnAllocator.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation; + +/** + * Allocates literal-derived columns in the aggregate's input: constants for aggregate + * calls whose arguments are literals, and {@code COALESCE(field, value)} columns for the + * {@code missing} request parameter. Calcite {@code AggregateCall} arguments are input + * column indices, so both kinds must exist as projected columns below the + * {@code LogicalAggregate}; the converter materializes every allocated column in a + * pre-aggregate {@code LogicalProject}. + */ +public interface LiteralColumnAllocator { + + /** + * Returns the input column index that will carry the given constant, allocating a new + * column on first use. Equal values share one column. + * + * @param value the literal value + * @return the column index in the aggregate's input row + */ + int columnFor(double value); + + /** + * Returns the input column index that will carry the given exact-integer constant, + * allocating a new column on first use. Equal values share one column. + * + * @param value the literal value + * @return the column index in the aggregate's input row + */ + int integerColumnFor(long value); + + /** + * Returns the input column index that will carry {@code COALESCE(field, missingValue)}, + * allocating a new column on first use. Equal (field, value) pairs share one column. + * + * @param fieldIndex input index of the field to coalesce + * @param missingValue substitute for SQL NULL (the {@code missing} request parameter) + * @return the column index in the aggregate's input row + */ + int coalescedColumnFor(int fieldIndex, double missingValue); +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslator.java index 29a0e48556f21..00564cda8a3c8 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslator.java @@ -8,14 +8,24 @@ package org.opensearch.dsl.aggregation.bucket; +import org.apache.lucene.util.BytesRef; import org.opensearch.dsl.aggregation.FieldGrouping; import org.opensearch.dsl.aggregation.GroupingInfo; import org.opensearch.dsl.result.BucketEntry; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.bucket.terms.DoubleTerms; +import org.opensearch.search.aggregations.bucket.terms.LongTerms; +import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregator; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.List; @@ -48,9 +58,104 @@ public BucketOrder getBucketOrder(TermsAggregationBuilder agg) { return agg.order(); } - // TODO: implement response conversion + /** + * Builds the terms response with classic-path key typing: integral keys → {@link LongTerms}, + * floating → {@link DoubleTerms}, booleans → {@link LongTerms} with the BOOLEAN format, + * binary (ip) keys decode to address strings, else {@link StringTerms}. Shard accounting + * fields are zero — the analytics path computes exact groups with no per-shard truncation. + */ @Override public InternalAggregation toBucketAggregation(TermsAggregationBuilder agg, Iterable buckets) { - throw new UnsupportedOperationException("toBucketAggregation not yet implemented"); + // SQL GROUP BY emits a NULL group; legacy terms excludes docs with a missing + // field entirely (no bucket) unless "missing" is configured. + List nonNull = new ArrayList<>(); + for (BucketEntry entry : buckets) { + if (entry.keys().get(0) != null) { + nonNull.add(entry); + } + } + Object sample = nonNull.isEmpty() ? null : nonNull.get(0).keys().get(0); + if (sample instanceof Boolean) { + return longTerms(agg, nonNull, DocValueFormat.BOOLEAN); + } + if (sample instanceof Double || sample instanceof Float) { + return doubleTerms(agg, nonNull); + } + if (sample instanceof Number) { + return longTerms(agg, nonNull, DocValueFormat.RAW); + } + return stringTerms(agg, nonNull); + } + + private static InternalAggregation stringTerms(TermsAggregationBuilder agg, List entries) { + List termBuckets = new ArrayList<>(entries.size()); + for (BucketEntry entry : entries) { + BytesRef term = new BytesRef(keyString(entry.keys().get(0))); + termBuckets.add(new StringTerms.Bucket(term, entry.docCount(), entry.subAggs(), false, 0, DocValueFormat.RAW)); + } + BucketOrder order = agg.order(); + return new StringTerms( + agg.getName(), + order, + order, + null, + DocValueFormat.RAW, + agg.shardSize(), + false, + 0, + termBuckets, + 0, + thresholds(agg) + ); + } + + private static InternalAggregation longTerms(TermsAggregationBuilder agg, List entries, DocValueFormat format) { + List termBuckets = new ArrayList<>(entries.size()); + for (BucketEntry entry : entries) { + Object key = entry.keys().get(0); + long term = key instanceof Boolean bool ? (bool ? 1L : 0L) : ((Number) key).longValue(); + termBuckets.add(new LongTerms.Bucket(term, entry.docCount(), entry.subAggs(), false, 0, format)); + } + BucketOrder order = agg.order(); + return new LongTerms(agg.getName(), order, order, null, format, agg.shardSize(), false, 0, termBuckets, 0, thresholds(agg)); + } + + private static InternalAggregation doubleTerms(TermsAggregationBuilder agg, List entries) { + List termBuckets = new ArrayList<>(entries.size()); + for (BucketEntry entry : entries) { + double term = ((Number) entry.keys().get(0)).doubleValue(); + termBuckets.add(new DoubleTerms.Bucket(term, entry.docCount(), entry.subAggs(), false, 0, DocValueFormat.RAW)); + } + BucketOrder order = agg.order(); + return new DoubleTerms( + agg.getName(), + order, + order, + null, + DocValueFormat.RAW, + agg.shardSize(), + false, + 0, + termBuckets, + 0, + thresholds(agg) + ); + } + + /** Binary keys are ip columns: render the address string like classic ip terms. */ + private static String keyString(Object key) { + if (key instanceof byte[] bytes) { + try { + return InetAddress.getByAddress(bytes).getHostAddress(); + } catch (UnknownHostException e) { + // Not a 4/16-byte address; fall back to a printable, deterministic form. + return Base64.getEncoder().encodeToString(bytes); + } + } + return key.toString(); + } + + private static TermsAggregator.BucketCountThresholds thresholds(TermsAggregationBuilder agg) { + return new TermsAggregator.BucketCountThresholds(agg.minDocCount(), agg.shardMinDocCount(), agg.size(), agg.shardSize()); } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AbstractMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AbstractMetricTranslator.java index d49570c0122b2..e93e1f2fdbd72 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AbstractMetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AbstractMetricTranslator.java @@ -13,18 +13,21 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.SqlAggFunction; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; import org.opensearch.dsl.converter.ConversionException; -import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import java.util.Collections; +import java.util.List; +import java.util.Map; /** - * Base class for metric translators. Provides the common {@link #toAggregateCall} - * logic — subclasses supply the SQL aggregate function, field name, and optionally - * override the return type. + * Base class for simple metric translators (single value: AVG, SUM, MIN, MAX, COUNT). + * Provides default implementations for single-value metrics, including the shared + * {@code missing} handling (aggregate over {@code COALESCE(field, missing)}) and + * {@code format} validation. */ -public abstract class AbstractMetricTranslator implements MetricTranslator { +public abstract class AbstractMetricTranslator> implements MetricTranslator { /** Creates a metric translator. */ protected AbstractMetricTranslator() {} @@ -40,36 +43,84 @@ protected AbstractMetricTranslator() {} */ protected abstract String getFieldName(T agg); + /** Whether the field must be numeric; count-like metrics override to accept any type. */ + protected boolean requiresNumericField() { + return true; + } + + /** Literal-free variant; {@code missing} needs the {@link LiteralColumnAllocator} variant. */ + @Override + public List toAggregateCalls(T agg, RelDataType rowType) throws ConversionException { + if (agg.missing() != null) { + throw new ConversionException("aggregation [" + agg.getName() + "] with a missing value requires literal column support"); + } + return toAggregateCalls(agg, rowType, null); + } + @Override - public AggregateCall toAggregateCall(T agg, RelDataType rowType) throws ConversionException { + public List toAggregateCalls(T agg, RelDataType rowType, LiteralColumnAllocator literals) throws ConversionException { + MetricTranslator.validateFormat(agg.format(), agg.getName()); String fieldName = getFieldName(agg); - RelDataTypeField field = rowType.getField(fieldName, false, false); - if (field == null) { - throw new ConversionException("Aggregation field '" + fieldName + "' not found in schema"); + RelDataTypeField field; + if (requiresNumericField()) { + field = MetricTranslator.resolveNumericField(rowType, fieldName, agg.getType()); + } else { + field = rowType.getField(fieldName, false, false); + if (field == null) { + throw new ConversionException("Aggregation field '" + fieldName + "' not found in schema"); + } } // Calcite enforces the return type to be same as input type; eg: AVG int→double coercion happens in response layer. - return AggregateCall.create( + AggregateCall call = AggregateCall.create( getAggFunction(), false, false, false, - Collections.singletonList(field.getIndex()), + Collections.singletonList(inputColumn(agg, field, literals)), -1, RelCollations.EMPTY, field.getType(), agg.getName() ); + return Collections.singletonList(call); } - @Override - public String getAggregateFieldName(T agg) { - return agg.getName(); + /** + * The aggregate's input column: the field, or {@code COALESCE(field, missing)} when the + * request sets {@code missing}. The coalesced column keeps the field's value type, so + * declared call types are unaffected. {@code literals} may be null only when missing is unset. + */ + static int inputColumn(ValuesSourceAggregationBuilder agg, RelDataTypeField field, LiteralColumnAllocator literals) + throws ConversionException { + if (agg.missing() == null) { + return field.getIndex(); + } + return literals.coalescedColumnFor(field.getIndex(), MetricTranslator.missingValue(agg.missing(), agg.getName())); } - // TODO: implement response conversion per metric type (InternalAvg, InternalSum, etc.) @Override - public InternalAggregation toInternalAggregation(String name, Object value) { - throw new UnsupportedOperationException("toInternalAggregation not yet implemented for " + getClass().getSimpleName()); + public List getAggregateFieldNames(T agg) { + return Collections.singletonList(agg.getName()); + } + + /** Extracts this metric's single output cell ({@code null} when the map is null or the cell is SQL NULL). */ + protected Object singleValue(T agg, Map values) { + return values == null ? null : values.get(agg.getName()); + } + + /** + * Coerces an engine result cell to double. Calcite keeps the input column type (AVG over + * an INTEGER column returns an integral value), so the int→double widening happens here. + * + * @param value the raw cell value (must be a {@link Number}) + */ + protected static double toDouble(Object value) { + if (value instanceof Number number) { + return number.doubleValue(); + } + throw new IllegalStateException( + "Expected numeric aggregation result but got " + (value == null ? "null" : value.getClass().getSimpleName()) + ); } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AvgMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AvgMetricTranslator.java index caa9eebe7daf4..1242f24a741cd 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AvgMetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/AvgMetricTranslator.java @@ -10,7 +10,12 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.search.DocValueFormat; +import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.opensearch.search.aggregations.metrics.InternalAvg; + +import java.util.Map; /** Translates AVG metric aggregation to Calcite. */ public class AvgMetricTranslator extends AbstractMetricTranslator { @@ -32,4 +37,19 @@ protected SqlAggFunction getAggFunction() { protected String getFieldName(AvgAggregationBuilder agg) { return agg.field(); } + + /** + * The engine returns the final average; encoded as (sum=value, count=1) so + * {@code InternalAvg.getValue()} reproduces it. Null (no matching docs) uses + * count=0, which renders as {@code "value": null} like legacy. + */ + @Override + public InternalAggregation toInternalAggregation(AvgAggregationBuilder agg, Map values) { + DocValueFormat format = MetricTranslator.parseFormat(agg.format()); + Object value = singleValue(agg, values); + if (value == null) { + return new InternalAvg(agg.getName(), 0.0, 0, format, null); + } + return new InternalAvg(agg.getName(), toDouble(value), 1, format, null); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslator.java new file mode 100644 index 0000000000000..cd01e79c76a76 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslator.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.InternalExtendedStats; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.COUNT_SUFFIX; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.MAX_SUFFIX; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.MIN_SUFFIX; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.SUM_SUFFIX; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.createCall; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.doubleValue; +import static org.opensearch.dsl.aggregation.metric.StatsMetricTranslator.longValue; + +/** + * Translator for the extended_stats aggregation: the four stats calls plus VAR_POP. + * sum_of_squares is derived from population variance (VAR_POP = E[x²] − mean², so + * SS = (variance + avg²) × count) because the engine has no SUM(x²) aggregate. + * avg, std_deviation, and std_deviation_bounds are derived by + * {@link InternalExtendedStats} from count/sum/variance and the request's sigma. + */ +public class ExtendedStatsMetricTranslator implements MetricTranslator { + + static final String VARIANCE_SUFFIX = "_variance"; + + /** Creates an extended_stats metric translator. */ + public ExtendedStatsMetricTranslator() {} + + @Override + public Class getAggregationType() { + return ExtendedStatsAggregationBuilder.class; + } + + /** See {@link AbstractMetricTranslator#toAggregateCalls(ValuesSourceAggregationBuilder, RelDataType)}. */ + @Override + public List toAggregateCalls(ExtendedStatsAggregationBuilder agg, RelDataType rowType) throws ConversionException { + if (agg.missing() != null) { + throw new ConversionException("aggregation [" + agg.getName() + "] with a missing value requires literal column support"); + } + return toAggregateCalls(agg, rowType, null); + } + + @Override + public List toAggregateCalls(ExtendedStatsAggregationBuilder agg, RelDataType rowType, LiteralColumnAllocator literals) + throws ConversionException { + MetricTranslator.validateFormat(agg.format(), agg.getName()); + RelDataTypeField field = MetricTranslator.resolveNumericField(rowType, agg.field(), agg.getType()); + int inputColumn = AbstractMetricTranslator.inputColumn(agg, field, literals); + String baseName = agg.getName(); + + List calls = new ArrayList<>(5); + calls.add(createCall(SqlStdOperatorTable.COUNT, inputColumn, field.getType(), baseName + COUNT_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.MIN, inputColumn, field.getType(), baseName + MIN_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.MAX, inputColumn, field.getType(), baseName + MAX_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.SUM, inputColumn, field.getType(), baseName + SUM_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.VAR_POP, inputColumn, field.getType(), baseName + VARIANCE_SUFFIX)); + return calls; + } + + @Override + public List getAggregateFieldNames(ExtendedStatsAggregationBuilder agg) { + String baseName = agg.getName(); + return List.of( + baseName + COUNT_SUFFIX, + baseName + MIN_SUFFIX, + baseName + MAX_SUFFIX, + baseName + SUM_SUFFIX, + baseName + VARIANCE_SUFFIX + ); + } + + @Override + public InternalAggregation toInternalAggregation(ExtendedStatsAggregationBuilder agg, Map values) { + String name = agg.getName(); + double sigma = agg.sigma(); + long count = longValue(values, name + COUNT_SUFFIX); + double min = doubleValue(values, name + MIN_SUFFIX, Double.POSITIVE_INFINITY); + double max = doubleValue(values, name + MAX_SUFFIX, Double.NEGATIVE_INFINITY); + double sum = doubleValue(values, name + SUM_SUFFIX, 0.0); + double variance = doubleValue(values, name + VARIANCE_SUFFIX, 0.0); + + double avg = count > 0 ? sum / count : 0; + double sumOfSquares = count > 0 ? (variance + avg * avg) * count : 0; + + return new InternalExtendedStats(name, count, sum, min, max, sumOfSquares, sigma, MetricTranslator.parseFormat(agg.format()), null); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentiles.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentiles.java new file mode 100644 index 0000000000000..fa5b15d53bdfd --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentiles.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.DocValueFormat; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalNumericMetricsAggregation; +import org.opensearch.search.aggregations.metrics.Percentile; +import org.opensearch.search.aggregations.metrics.Percentiles; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * Percentiles response holding engine-computed final values — the server's + * {@code InternalTDigestPercentiles} wraps a live digest, which this path does not have. + * Renders legacy-identical JSON (keyed and non-keyed, {@code _as_string} for non-RAW + * formats) and reports the legacy type name for {@code typed_keys}. Never crosses the + * wire on this path; wire methods exist only to satisfy {@link InternalAggregation}. + */ +public class InternalDslPercentiles extends InternalNumericMetricsAggregation.MultiValue implements Percentiles { + + /** Matches the legacy tdigest percentiles type for typed_keys response parity. */ + static final String TYPE_NAME = "tdigest_percentiles"; + + private final double[] percents; + private final double[] values; // NaN = no value (empty result set) + private final boolean keyed; + + /** + * Creates a percentiles response. + * + * @param name the aggregation name + * @param percents requested percents, in request order + * @param values one value per percent ({@code NaN} when execution produced none) + * @param keyed whether to render the keyed (object) or non-keyed (array) form + * @param format value format; non-RAW formats add {@code _as_string} companions + */ + public InternalDslPercentiles(String name, double[] percents, double[] values, boolean keyed, DocValueFormat format) { + super(name, null); + if (percents.length != values.length) { + throw new IllegalArgumentException("percents and values length mismatch"); + } + this.percents = percents; + this.values = values; + this.keyed = keyed; + this.format = Objects.requireNonNull(format, "format must not be null"); + } + + @Override + public String getWriteableName() { + return TYPE_NAME; + } + + @Override + public double percentile(double percent) { + for (int i = 0; i < percents.length; i++) { + if (percents[i] == percent) { + return values[i]; + } + } + throw new IllegalArgumentException("percent requested [" + percent + "] was not computed"); + } + + @Override + public String percentileAsString(double percent) { + return format.format(percentile(percent)).toString(); + } + + @Override + public double value(String name) { + return percentile(Double.parseDouble(name)); + } + + @Override + public Iterator iterator() { + return new Iterator<>() { + private int i = 0; + + @Override + public boolean hasNext() { + return i < percents.length; + } + + @Override + public Percentile next() { + Percentile p = new Percentile(percents[i], values[i]); + i++; + return p; + } + }; + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (keyed) { + builder.startObject(CommonFields.VALUES.getPreferredName()); + for (int i = 0; i < percents.length; i++) { + String key = String.valueOf(percents[i]); + builder.field(key, Double.isNaN(values[i]) ? null : values[i]); + if (format != DocValueFormat.RAW && Double.isNaN(values[i]) == false) { + builder.field(key + "_as_string", format.format(values[i]).toString()); + } + } + builder.endObject(); + } else { + builder.startArray(CommonFields.VALUES.getPreferredName()); + for (int i = 0; i < percents.length; i++) { + builder.startObject(); + builder.field(CommonFields.KEY.getPreferredName(), percents[i]); + builder.field(CommonFields.VALUE.getPreferredName(), Double.isNaN(values[i]) ? null : values[i]); + if (format != DocValueFormat.RAW && Double.isNaN(values[i]) == false) { + builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(values[i]).toString()); + } + builder.endObject(); + } + builder.endArray(); + } + return builder; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeDoubleArray(percents); + out.writeDoubleArray(values); + out.writeBoolean(keyed); + } + + @Override + public InternalAggregation reduce(List aggregations, ReduceContext reduceContext) { + throw new UnsupportedOperationException("percentiles are not reduced on the DSL analytics path"); + } + + @Override + protected boolean mustReduceOnSingleInternalAgg() { + return false; + } + + @Override + public Object getProperty(List path) { + if (path.isEmpty()) { + return this; + } + if (path.size() == 1) { + return value(path.get(0)); + } + throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Arrays.hashCode(percents), Arrays.hashCode(values), keyed); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (super.equals(obj) == false) return false; + InternalDslPercentiles other = (InternalDslPercentiles) obj; + return keyed == other.keyed && Arrays.equals(percents, other.percents) && Arrays.equals(values, other.values); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MaxMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MaxMetricTranslator.java index cda4b97690f13..d640a9478162d 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MaxMetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MaxMetricTranslator.java @@ -10,8 +10,12 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; +import java.util.Map; + /** Translates MAX metric aggregation to Calcite. */ public class MaxMetricTranslator extends AbstractMetricTranslator { @@ -32,4 +36,12 @@ protected SqlAggFunction getAggFunction() { protected String getFieldName(MaxAggregationBuilder agg) { return agg.field(); } + + /** Null (no matching docs) becomes -Infinity — legacy sentinel, rendered as {@code "value": null}. */ + @Override + public InternalAggregation toInternalAggregation(MaxAggregationBuilder agg, Map values) { + Object value = singleValue(agg, values); + double max = value == null ? Double.NEGATIVE_INFINITY : toDouble(value); + return new InternalMax(agg.getName(), max, MetricTranslator.parseFormat(agg.format()), null); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MetricTranslator.java index e0a67718ea9d4..be9fe9df40c34 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MetricTranslator.java @@ -10,44 +10,147 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.dsl.aggregation.AggregationTranslator; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.InternalAggregation; +import java.util.List; +import java.util.Map; + /** - * Translates a metric aggregation (AVG, SUM, MIN, MAX, etc.) to a Calcite AggregateCall, + * Translates a metric aggregation to Calcite AggregateCall(s), * and converts raw result values back to OpenSearch InternalAggregation for response building. */ public interface MetricTranslator extends AggregationTranslator { /** - * Converts the metric aggregation to a Calcite AggregateCall. + * Resolves {@code fieldName} and requires a numeric column type, rejecting others with the + * classic path's {@code illegal_argument_exception} shape. Date and boolean columns are + * rejected too (unlike classic search) until the analytics path implements them. + * + * @param rowType the index row type + * @param fieldName the aggregated field + * @param aggregationType the request aggregation type name for the error message (e.g. "avg") + * @return the resolved numeric field + * @throws ConversionException if the field does not exist + */ + static RelDataTypeField resolveNumericField(RelDataType rowType, String fieldName, String aggregationType) throws ConversionException { + RelDataTypeField field = rowType.getField(fieldName, false, false); + if (field == null) { + throw new ConversionException("Aggregation field '" + fieldName + "' not found in schema"); + } + SqlTypeName type = field.getType().getSqlTypeName(); + if (SqlTypeName.NUMERIC_TYPES.contains(type) == false) { + throw new IllegalArgumentException( + "Field [" + fieldName + "] of type [" + type + "] is not supported for aggregation [" + aggregationType + "]" + ); + } + return field; + } + + /** + * Resolves the request's {@code format} pattern to a {@link DocValueFormat} (decimal + * patterns only; RAW when absent). Validate first via {@link #validateFormat}. + * + * @param pattern the request's format pattern, or null/empty for RAW + * @return the resolved format + */ + static DocValueFormat parseFormat(String pattern) { + if (pattern == null || pattern.isEmpty()) { + return DocValueFormat.RAW; + } + return new DocValueFormat.Decimal(pattern); + } + + /** + * Validates the {@code format} pattern so an invalid pattern fails at conversion, + * before any engine work. + * + * @param pattern the request's format pattern + * @param aggName the aggregation name for the error message + * @throws ConversionException if the pattern is not a valid decimal format + */ + static void validateFormat(String pattern, String aggName) throws ConversionException { + try { + parseFormat(pattern); + } catch (IllegalArgumentException e) { + throw new ConversionException("aggregation [" + aggName + "] has an invalid format [" + pattern + "]: " + e.getMessage()); + } + } + + /** + * Coerces the request's {@code missing} value to a double for the COALESCE substitute + * column; non-numeric substitutes are rejected. + * + * @param missing the request's missing value (non-null) + * @param aggName the aggregation name for the error message + * @return the substitute as a double + * @throws ConversionException if the value is not a number or numeric string + */ + static double missingValue(Object missing, String aggName) throws ConversionException { + if (missing instanceof Number number) { + return number.doubleValue(); + } + if (missing instanceof String s) { + try { + return Double.parseDouble(s); + } catch (NumberFormatException e) { + // fall through to the shared error below + } + } + throw new ConversionException("aggregation [" + aggName + "] has a non-numeric missing value [" + missing + "]"); + } + + /** + * Converts the metric aggregation to Calcite AggregateCall(s). + * + * @param agg the metric aggregation builder + * @param rowType the index row type for field lookup + * @return list of Calcite AggregateCalls + * @throws ConversionException if conversion fails + */ + List toAggregateCalls(T agg, RelDataType rowType) throws ConversionException; + + /** + * Variant for metrics whose aggregate calls take literal arguments: + * constants allocated through {@code literals} become input columns of the aggregate. + * Metrics without literal arguments need not override this. * * @param agg the metric aggregation builder * @param rowType the index row type for field lookup - * @return the Calcite AggregateCall + * @param literals allocator for constant input columns + * @return list of Calcite AggregateCalls * @throws ConversionException if conversion fails */ - AggregateCall toAggregateCall(T agg, RelDataType rowType) throws ConversionException; + default List toAggregateCalls(T agg, RelDataType rowType, LiteralColumnAllocator literals) throws ConversionException { + return toAggregateCalls(agg, rowType); + } /** - * Returns the output field name for this aggregation. + * Returns the output field names for this aggregation. * * @param agg the metric aggregation builder - * @return the aggregate field name + * @return list of aggregate field names */ - String getAggregateFieldName(T agg); + List getAggregateFieldNames(T agg); // TODO: Revisit signature — accept a stream/iterator of for bulk conversion // to avoid per-row virtual dispatch overhead, and use Arrow-native types once Analytics Core // exposes them. /** - * Converts a raw result value from execution into an OpenSearch InternalAggregation. + * Converts raw result values from execution into an OpenSearch InternalAggregation. + * The builder is passed (not just the name) so translators can honor request + * parameters that shape the response, e.g. extended_stats sigma. * - * @param name the aggregation name - * @param value the raw value (may be null) + * @param agg the original aggregation builder from the request + * @param values values keyed by {@link #getAggregateFieldNames} entries; {@code null} + * when execution produced no row for this metric's granularity * @return the corresponding InternalAggregation */ - InternalAggregation toInternalAggregation(String name, Object value); + InternalAggregation toInternalAggregation(T agg, Map values); } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MinMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MinMetricTranslator.java index 080b139591c97..010bfd49a58b9 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MinMetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/MinMetricTranslator.java @@ -10,8 +10,12 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalMin; import org.opensearch.search.aggregations.metrics.MinAggregationBuilder; +import java.util.Map; + /** Translates MIN metric aggregation to Calcite. */ public class MinMetricTranslator extends AbstractMetricTranslator { @@ -32,4 +36,12 @@ protected SqlAggFunction getAggFunction() { protected String getFieldName(MinAggregationBuilder agg) { return agg.field(); } + + /** Null (no matching docs) becomes +Infinity — legacy sentinel, rendered as {@code "value": null}. */ + @Override + public InternalAggregation toInternalAggregation(MinAggregationBuilder agg, Map values) { + Object value = singleValue(agg, values); + double min = value == null ? Double.POSITIVE_INFINITY : toDouble(value); + return new InternalMin(agg.getName(), min, MetricTranslator.parseFormat(agg.format()), null); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentileApproxFunction.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentileApproxFunction.java new file mode 100644 index 0000000000000..b1998334db6ea --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentileApproxFunction.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeTransforms; +import org.apache.calcite.util.Optionality; + +/** + * Aggregate function stubs for {@code PERCENTILE_APPROX(field, percent)} and the + * explicit-compression {@code PERCENTILE_APPROX_N(field, percent, centroids)}. The + * DataFusion backend's rewriter matches both by name and binds them to + * {@code approx_percentile_cont}, rescaling the percent literal to [0, 1] at emission. + */ +final class PercentileApproxFunction extends SqlAggFunction { + + /** Two-arg form: {@code PERCENTILE_APPROX(field, percent)} — engine-default compression. */ + static final PercentileApproxFunction INSTANCE = new PercentileApproxFunction("PERCENTILE_APPROX", OperandTypes.ANY_ANY); + + /** Three-arg form: {@code PERCENTILE_APPROX_N(field, percent, centroids)}. */ + static final PercentileApproxFunction INSTANCE_N = new PercentileApproxFunction( + "PERCENTILE_APPROX_N", + OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.ANY, SqlTypeFamily.ANY) + ); + + private PercentileApproxFunction(String name, SqlOperandTypeChecker operandTypeChecker) { + super( + name, + null, + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0.andThen(SqlTypeTransforms.FORCE_NULLABLE), + null, + operandTypeChecker, + SqlFunctionCategory.USER_DEFINED_FUNCTION, + false, + false, + Optionality.FORBIDDEN + ); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslator.java new file mode 100644 index 0000000000000..b49fc6e702db0 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslator.java @@ -0,0 +1,144 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.opensearch.search.aggregations.metrics.PercentilesConfig; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Translator for the percentiles aggregation. One request node fans out to one + * {@code PERCENTILE_APPROX(field, percent)} call per requested percent, bound by the + * DataFusion backend to {@code approx_percentile_cont} (t-digest, so values are + * approximate like legacy). Honors {@code missing}, {@code format}, and tdigest + * {@code compression} (as the engine's centroids arg, 3-arg {@code PERCENTILE_APPROX_N}). + * HDR is unsupported and fails conversion — there is no classic-path fallback. + */ +public class PercentilesMetricTranslator implements MetricTranslator { + + /** Creates a percentiles metric translator. */ + public PercentilesMetricTranslator() {} + + @Override + public Class getAggregationType() { + return PercentilesAggregationBuilder.class; + } + + @Override + public List toAggregateCalls(PercentilesAggregationBuilder agg, RelDataType rowType) throws ConversionException { + throw new ConversionException("percentiles requires literal column support; use the LiteralColumnAllocator variant"); + } + + @Override + public List toAggregateCalls(PercentilesAggregationBuilder agg, RelDataType rowType, LiteralColumnAllocator literals) + throws ConversionException { + validate(agg); + RelDataTypeField field = MetricTranslator.resolveNumericField(rowType, agg.field(), agg.getType()); + + int inputColumn = field.getIndex(); + if (agg.missing() != null) { + inputColumn = literals.coalescedColumnFor(field.getIndex(), MetricTranslator.missingValue(agg.missing(), agg.getName())); + } + + Long centroidCount = centroids(agg); + Integer centroidsColumn = centroidCount == null ? null : literals.integerColumnFor(centroidCount); + + // Declared with the field's type; both percentile ops infer ARG0 forced nullable, and the + // metadata builder normalizes the declared type to nullable to match (it owns the factory). + List calls = new ArrayList<>(agg.percentiles().length); + for (double percent : agg.percentiles()) { + int percentColumn = literals.columnFor(percent); + calls.add( + AggregateCall.create( + centroidsColumn == null ? PercentileApproxFunction.INSTANCE : PercentileApproxFunction.INSTANCE_N, + false, + true, + false, + centroidsColumn == null ? List.of(inputColumn, percentColumn) : List.of(inputColumn, percentColumn, centroidsColumn), + -1, + RelCollations.EMPTY, + field.getType(), + columnName(agg.getName(), percent) + ) + ); + } + return calls; + } + + /** Vanilla's default t-digest compression; also the engine's default centroid count. */ + private static final double DEFAULT_COMPRESSION = 100.0; + + /** + * Centroids for a non-default tdigest compression, or {@code null} for the engine default. + * Config presence is not the signal — the request parser injects {@code TDigest(100)} when + * the JSON names no method — so only a non-default compression pins the value in the plan. + */ + private static Long centroids(PercentilesAggregationBuilder agg) throws ConversionException { + if (agg.percentilesConfig() instanceof PercentilesConfig.TDigest tdigest && tdigest.getCompression() != DEFAULT_COMPRESSION) { + long centroidCount = Math.round(tdigest.getCompression()); + if (centroidCount < 1) { + throw new ConversionException( + "percentiles aggregation [" + + agg.getName() + + "] has compression [" + + tdigest.getCompression() + + "]; the analytics path requires at least 1" + ); + } + return centroidCount; + } + return null; + } + + @Override + public List getAggregateFieldNames(PercentilesAggregationBuilder agg) { + List names = new ArrayList<>(agg.percentiles().length); + for (double percent : agg.percentiles()) { + names.add(columnName(agg.getName(), percent)); + } + return names; + } + + @Override + public InternalAggregation toInternalAggregation(PercentilesAggregationBuilder agg, Map values) { + double[] percents = agg.percentiles(); + double[] results = new double[percents.length]; + for (int i = 0; i < percents.length; i++) { + Object cell = values == null ? null : values.get(columnName(agg.getName(), percents[i])); + results[i] = cell == null ? Double.NaN : ((Number) cell).doubleValue(); + } + return new InternalDslPercentiles(agg.getName(), percents, results, agg.keyed(), MetricTranslator.parseFormat(agg.format())); + } + + private static void validate(PercentilesAggregationBuilder agg) throws ConversionException { + if (agg.percentiles() == null || agg.percentiles().length == 0) { + throw new ConversionException("percentiles aggregation [" + agg.getName() + "] has no percents"); + } + PercentilesConfig config = agg.percentilesConfig(); + if (config instanceof PercentilesConfig.Hdr) { + throw new ConversionException("HDR percentiles method is not supported on the analytics path"); + } + MetricTranslator.validateFormat(agg.format(), agg.getName()); + } + + /** Output column name for one percent, e.g. {@code lat_p50_0} for percent 50.0. */ + static String columnName(String aggName, double percent) { + return aggName + "_p" + String.valueOf(percent).replace('.', '_').replace('-', '_'); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslator.java new file mode 100644 index 0000000000000..4a37d453ec67b --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslator.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalStats; +import org.opensearch.search.aggregations.metrics.StatsAggregationBuilder; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Translator for the stats aggregation. One request node fans out to four SQL aggregate + * calls (COUNT, MIN, MAX, SUM), output columns named {@code _}. + * avg is not computed by the engine — {@link InternalStats} derives it as sum/count. + */ +public class StatsMetricTranslator implements MetricTranslator { + + static final String COUNT_SUFFIX = "_count"; + static final String MIN_SUFFIX = "_min"; + static final String MAX_SUFFIX = "_max"; + static final String SUM_SUFFIX = "_sum"; + + /** Creates a stats metric translator. */ + public StatsMetricTranslator() {} + + @Override + public Class getAggregationType() { + return StatsAggregationBuilder.class; + } + + /** See {@link AbstractMetricTranslator#toAggregateCalls(ValuesSourceAggregationBuilder, RelDataType)}. */ + @Override + public List toAggregateCalls(StatsAggregationBuilder agg, RelDataType rowType) throws ConversionException { + if (agg.missing() != null) { + throw new ConversionException("aggregation [" + agg.getName() + "] with a missing value requires literal column support"); + } + return toAggregateCalls(agg, rowType, null); + } + + @Override + public List toAggregateCalls(StatsAggregationBuilder agg, RelDataType rowType, LiteralColumnAllocator literals) + throws ConversionException { + MetricTranslator.validateFormat(agg.format(), agg.getName()); + RelDataTypeField field = MetricTranslator.resolveNumericField(rowType, agg.field(), agg.getType()); + int inputColumn = AbstractMetricTranslator.inputColumn(agg, field, literals); + String baseName = agg.getName(); + + List calls = new ArrayList<>(4); + calls.add(createCall(SqlStdOperatorTable.COUNT, inputColumn, field.getType(), baseName + COUNT_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.MIN, inputColumn, field.getType(), baseName + MIN_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.MAX, inputColumn, field.getType(), baseName + MAX_SUFFIX)); + calls.add(createCall(SqlStdOperatorTable.SUM, inputColumn, field.getType(), baseName + SUM_SUFFIX)); + return calls; + } + + static AggregateCall createCall(SqlAggFunction function, int inputColumn, RelDataType type, String name) { + // Calcite enforces the return type to be same as input type; numeric widening happens in the response layer. + return AggregateCall.create( + function, + false, + false, + false, + Collections.singletonList(inputColumn), + -1, + RelCollations.EMPTY, + type, + name + ); + } + + @Override + public List getAggregateFieldNames(StatsAggregationBuilder agg) { + String baseName = agg.getName(); + return List.of(baseName + COUNT_SUFFIX, baseName + MIN_SUFFIX, baseName + MAX_SUFFIX, baseName + SUM_SUFFIX); + } + + @Override + public InternalAggregation toInternalAggregation(StatsAggregationBuilder agg, Map values) { + String name = agg.getName(); + long count = longValue(values, name + COUNT_SUFFIX); + double min = doubleValue(values, name + MIN_SUFFIX, Double.POSITIVE_INFINITY); + double max = doubleValue(values, name + MAX_SUFFIX, Double.NEGATIVE_INFINITY); + double sum = doubleValue(values, name + SUM_SUFFIX, 0.0); + return new InternalStats(name, count, sum, min, max, MetricTranslator.parseFormat(agg.format()), null); + } + + /** Reads a long cell; missing map or SQL NULL degrades to 0 (empty-set count). */ + static long longValue(Map values, String key) { + Object value = values == null ? null : values.get(key); + return value == null ? 0L : ((Number) value).longValue(); + } + + /** Reads a double cell; missing map or SQL NULL degrades to the empty-set sentinel. */ + static double doubleValue(Map values, String key, double emptyValue) { + Object value = values == null ? null : values.get(key); + return value == null ? emptyValue : ((Number) value).doubleValue(); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/SumMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/SumMetricTranslator.java index 6543a00d1aca0..16e964bbe4542 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/SumMetricTranslator.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/SumMetricTranslator.java @@ -10,8 +10,12 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalSum; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; +import java.util.Map; + /** Translates SUM metric aggregation to Calcite. */ public class SumMetricTranslator extends AbstractMetricTranslator { @@ -32,4 +36,12 @@ protected SqlAggFunction getAggFunction() { protected String getFieldName(SumAggregationBuilder agg) { return agg.field(); } + + /** Null (no matching docs) becomes 0.0 — legacy sum-of-nothing semantics. */ + @Override + public InternalAggregation toInternalAggregation(SumAggregationBuilder agg, Map values) { + Object value = singleValue(agg, values); + double sum = value == null ? 0.0 : toDouble(value); + return new InternalSum(agg.getName(), sum, MetricTranslator.parseFormat(agg.format()), null); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslator.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslator.java new file mode 100644 index 0000000000000..8e7674b268206 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslator.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalValueCount; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; + +import java.util.Map; + +/** + * Translator for value_count aggregation. + * Counts the number of values for a field. + */ +public class ValueCountMetricTranslator extends AbstractMetricTranslator { + + /** Creates a value_count metric translator. */ + public ValueCountMetricTranslator() {} + + @Override + public Class getAggregationType() { + return ValueCountAggregationBuilder.class; + } + + @Override + protected SqlAggFunction getAggFunction() { + return SqlStdOperatorTable.COUNT; + } + + @Override + protected String getFieldName(ValueCountAggregationBuilder agg) { + return agg.field(); + } + + /** value_count only tests value presence — any field type counts, matching classic search. */ + @Override + protected boolean requiresNumericField() { + return false; + } + + /** Null (no matching docs) becomes 0 — a count over nothing is zero. */ + @Override + public InternalAggregation toInternalAggregation(ValueCountAggregationBuilder agg, Map values) { + Object value = singleValue(agg, values); + long count = value == null ? 0 : ((Number) value).longValue(); + return new InternalValueCount(agg.getName(), count, null); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/AggregateConverter.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/AggregateConverter.java index 0714a3e058030..56f9d8a9f131c 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/AggregateConverter.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/AggregateConverter.java @@ -10,11 +10,26 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.dsl.aggregation.AggregationMetadata; +import org.opensearch.dsl.aggregation.LiteralColumn; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; /** * Creates a {@link LogicalAggregate} from pre-computed {@link AggregationMetadata}. * The metadata is produced by the tree walker and set on the context before this runs. + * + *

When the metadata carries literal-derived columns (constant call arguments or a + * {@code missing} substitution as {@code COALESCE(field, value)}), the input is wrapped in + * an identity project appending one column per entry; the identity prefix preserves input + * positions, so group-by indices are unaffected. */ public class AggregateConverter { @@ -29,6 +44,42 @@ public AggregateConverter() {} * @return the LogicalAggregate node */ public RelNode convert(RelNode input, AggregationMetadata metadata) { - return LogicalAggregate.create(input, metadata.getGroupByBitSet(), null, metadata.getAggregateCalls()); + RelNode aggInput = metadata.getLiteralColumns().isEmpty() ? input : appendLiteralColumns(input, metadata.getLiteralColumns()); + return LogicalAggregate.create(aggInput, metadata.getGroupByBitSet(), null, metadata.getAggregateCalls()); + } + + private static RelNode appendLiteralColumns(RelNode input, List literals) { + RexBuilder rexBuilder = input.getCluster().getRexBuilder(); + int fieldCount = input.getRowType().getFieldCount(); + + List projects = new ArrayList<>(fieldCount + literals.size()); + List names = new ArrayList<>(fieldCount + literals.size()); + for (int i = 0; i < fieldCount; i++) { + projects.add(rexBuilder.makeInputRef(input, i)); + names.add(input.getRowType().getFieldNames().get(i)); + } + for (int i = 0; i < literals.size(); i++) { + LiteralColumn column = literals.get(i); + switch (column.kind()) { + case DOUBLE_CONSTANT -> { + projects.add(rexBuilder.makeApproxLiteral(BigDecimal.valueOf(column.value()))); + names.add("_lit" + i); + } + case INTEGER_CONSTANT -> { + projects.add(rexBuilder.makeExactLiteral(BigDecimal.valueOf((long) column.value()))); + names.add("_lit" + i); + } + case COALESCED -> { + // COALESCE operands must share one value type (backend CoalesceAdapter), so cast + // the substitute to the field's type; nullability is left to COALESCE itself. + RexNode fieldRef = rexBuilder.makeInputRef(input, column.coalesceFieldIndex()); + RexNode constant = rexBuilder.makeApproxLiteral(BigDecimal.valueOf(column.value())); + RexNode substitute = rexBuilder.makeCast(fieldRef.getType(), constant, false, false); + projects.add(rexBuilder.makeCall(SqlStdOperatorTable.COALESCE, fieldRef, substitute)); + names.add("_missing" + i); + } + } + } + return LogicalProject.create(input, List.of(), projects, names, Set.of()); } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/SearchSourceConverter.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/SearchSourceConverter.java index 21d0d60c4d8d5..8a5156ae2f932 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/SearchSourceConverter.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/converter/SearchSourceConverter.java @@ -23,6 +23,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.opensearch.dsl.aggregation.AggregationMetadata; +import org.opensearch.dsl.aggregation.AggregationRegistry; import org.opensearch.dsl.aggregation.AggregationRegistryFactory; import org.opensearch.dsl.aggregation.AggregationTreeWalker; import org.opensearch.dsl.executor.QueryPlans; @@ -50,6 +51,7 @@ public class SearchSourceConverter { private final AggregateConverter aggConverter; private final PostAggregateConverter postAggConverter; private final AggregationTreeWalker treeWalker; + private final AggregationRegistry aggRegistry; /** * Initializes planning infrastructure from the given schema. @@ -77,10 +79,15 @@ public SearchSourceConverter(SchemaPlus schema) { this.aggConverter = new AggregateConverter(); this.postAggConverter = new PostAggregateConverter(); - var aggRegistry = AggregationRegistryFactory.create(); + this.aggRegistry = AggregationRegistryFactory.create(); this.treeWalker = new AggregationTreeWalker(aggRegistry); } + /** Returns the aggregation registry used by this converter. */ + public AggregationRegistry getAggregationRegistry() { + return aggRegistry; + } + /** * Converts DSL for the given index into query plans. * diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/AggregationResponseBuilder.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/AggregationResponseBuilder.java new file mode 100644 index 0000000000000..ca4098eede3eb --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/AggregationResponseBuilder.java @@ -0,0 +1,317 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.result; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.opensearch.dsl.aggregation.AggregationRegistry; +import org.opensearch.dsl.aggregation.AggregationTranslator; +import org.opensearch.dsl.aggregation.GroupingInfo; +import org.opensearch.dsl.aggregation.bucket.BucketTranslator; +import org.opensearch.dsl.aggregation.metric.MetricTranslator; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.dsl.util.ComparisonUtils; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.InternalAggregations; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Converts execution results into OpenSearch InternalAggregations format. + * Uses granularity-based matching to map flat tabular results to hierarchical aggregation structures. + */ +public final class AggregationResponseBuilder { + + private static final String NO_GROUPING_KEY = ""; + /** NUL cannot appear in field names, so joined keys cannot collide. */ + private static final String AGGREGATION_LEVEL_SEPARATOR = "\u0000"; + + private final AggregationRegistry registry; + private final Map granularityMap; + + public AggregationResponseBuilder(AggregationRegistry registry, List aggResults) { + this.registry = registry; + this.granularityMap = new HashMap<>(); + for (ExecutionResult result : aggResults) { + String key = computeGranularityKey(result); + granularityMap.put(key, result); + } + } + + /** + * Builds InternalAggregations from the original aggregation builders. + */ + public InternalAggregations build(List originalAggs) throws ConversionException { + List aggs = buildLevel(originalAggs, new ArrayList<>(), Map.of()); + return InternalAggregations.from(aggs); + } + + /** + * Recursively builds aggregations at a specific nesting level. + * Routes to buildMetric or buildBucket based on aggregation type. + */ + private List buildLevel( + List aggs, + List accumulatedGroupFields, + Map parentKeyFilter + ) throws ConversionException { + + List result = new ArrayList<>(); + + for (AggregationBuilder agg : aggs) { + @SuppressWarnings("unchecked") + AggregationTranslator type = (AggregationTranslator) registry.get(agg.getClass()); + + if (type instanceof MetricTranslator) { + result.add(buildMetric((MetricTranslator) type, agg, accumulatedGroupFields, parentKeyFilter)); + } else if (type instanceof BucketTranslator) { + result.add(buildBucket((BucketTranslator) type, agg, accumulatedGroupFields, parentKeyFilter)); + } else { + throw new ConversionException( + "No response translator for aggregation [" + agg.getName() + "] of type " + agg.getClass().getSimpleName() + ); + } + } + return result; + } + + /** + * Builds a metric aggregation from execution results: finds the matching row via granularity + * key and parent filters, then assembles the translator's output columns (several for + * multi-column metrics like stats) into a values map keyed by {@code getAggregateFieldNames}. + */ + private InternalAggregation buildMetric( + MetricTranslator translator, + AggregationBuilder agg, + List accumulatedGroupFields, + Map parentKeyFilter + ) { + + ExecutionResult result = granularityMap.get(granularityKey(accumulatedGroupFields)); + + if (result == null) { + return buildEmptyMetric(translator, agg); + } + + List rows = StreamSupport.stream(result.getRows().spliterator(), false).collect(Collectors.toList()); + + if (rows.isEmpty()) { + return buildEmptyMetric(translator, agg); + } + + Map colIndex = buildColumnIndex(result); + Object[] matchingRow = findMatchingRow(rows, colIndex, parentKeyFilter); + + if (matchingRow == null) { + return buildEmptyMetric(translator, agg); + } + + Map values = new HashMap<>(); + for (String fieldName : translator.getAggregateFieldNames(agg)) { + Integer colIdx = colIndex.get(fieldName); + if (colIdx != null) { + values.put(fieldName, matchingRow[colIdx]); + } + } + + if (values.isEmpty()) { + return buildEmptyMetric(translator, agg); + } + return translator.toInternalAggregation(agg, values); + } + + /** + * Builds an empty metric aggregation with no computed value. + */ + private static InternalAggregation buildEmptyMetric(MetricTranslator translator, AggregationBuilder agg) { + return translator.toInternalAggregation(agg, null); + } + + /** + * Builds an empty bucket aggregation with no buckets. + */ + private static InternalAggregation buildEmptyBucket(BucketTranslator translator, AggregationBuilder agg) { + return translator.toBucketAggregation(agg, List.of()); + } + + /** + * Builds a bucket aggregation by grouping rows and recursively building sub-aggregations. + * Groups rows by bucket keys and recursively processes nested aggregations for each bucket. + */ + private InternalAggregation buildBucket( + BucketTranslator translator, + AggregationBuilder agg, + List accumulatedGroupFields, + Map parentKeyFilter + ) throws ConversionException { + + GroupingInfo grouping = translator.getGrouping(agg); + List newGroupFields = new ArrayList<>(accumulatedGroupFields); + newGroupFields.addAll(grouping.getFieldNames()); + + ExecutionResult result = granularityMap.get(granularityKey(newGroupFields)); + + if (result == null) { + return buildEmptyBucket(translator, agg); + } + + List rows = StreamSupport.stream(result.getRows().spliterator(), false).collect(Collectors.toList()); + + if (rows.isEmpty()) { + return buildEmptyBucket(translator, agg); + } + + Map colIndex = buildColumnIndex(result); + List filteredRows = filterRows(rows, colIndex, parentKeyFilter); + + List currentGroupColumns = new ArrayList<>(grouping.getFieldNames()); + + Map, List> grouped = groupRowsByKeys(filteredRows, currentGroupColumns, colIndex); + + List buckets = new ArrayList<>(); + List subAggs = new ArrayList<>(translator.getSubAggregations(agg)); + + for (Map.Entry, List> entry : grouped.entrySet()) { + Map childFilter = new HashMap<>(parentKeyFilter); + for (int i = 0; i < currentGroupColumns.size(); i++) { + childFilter.put(currentGroupColumns.get(i), entry.getKey().get(i)); + } + + Integer countIdx = colIndex.get("_count"); + if (countIdx == null) { + throw new ConversionException("Missing _count column in aggregation result"); + } + Object[] firstRowInGroup = entry.getValue().get(0); + long docCount = ((Number) firstRowInGroup[countIdx]).longValue(); + + InternalAggregations subAggregations = subAggs.isEmpty() + ? InternalAggregations.EMPTY + : InternalAggregations.from(buildLevel(subAggs, newGroupFields, childFilter)); + + buckets.add(new BucketEntry(entry.getKey(), docCount, subAggregations)); + } + + return translator.toBucketAggregation(agg, buckets); + } + + /** + * Builds a map from column names to their indices. + * Enables efficient column lookup by name during row processing. + */ + private static Map buildColumnIndex(ExecutionResult result) { + Map index = new HashMap<>(); + List fieldNames = result.getFieldNames(); + for (int i = 0; i < fieldNames.size(); i++) { + index.put(fieldNames.get(i), i); + } + return index; + } + + /** + * Finds the first row matching all filter criteria. + * Used to locate the specific row for nested metric aggregations. + */ + private static Object[] findMatchingRow(List rows, Map colIndex, Map filter) { + for (Object[] row : rows) { + if (matchesFilter(row, colIndex, filter)) { + return row; + } + } + return null; + } + + /** + * Filters rows to only those matching all filter criteria. + * Ensures nested aggregations only process rows belonging to their parent bucket. + */ + private static List filterRows(List rows, Map colIndex, Map filter) { + if (filter.isEmpty()) { + return rows; + } + return rows.stream().filter(row -> matchesFilter(row, colIndex, filter)).collect(Collectors.toList()); + } + + /** + * Checks if a row matches all filter criteria. + * Uses type-coercion comparison to handle numeric type differences from execution engine. + */ + private static boolean matchesFilter(Object[] row, Map colIndex, Map filter) { + for (Map.Entry entry : filter.entrySet()) { + Integer idx = colIndex.get(entry.getKey()); + if (idx == null || !ComparisonUtils.valuesEqual(row[idx], entry.getValue())) { + return false; + } + } + return true; + } + + /** + * Groups rows by their grouping column values (the bucket keys). Group columns are + * resolved by name — they are not guaranteed to be the leading row positions. + */ + private static Map, List> groupRowsByKeys( + List rows, + List groupColumns, + Map colIndex + ) throws ConversionException { + List keyIndices = new ArrayList<>(groupColumns.size()); + for (String column : groupColumns) { + Integer idx = colIndex.get(column); + if (idx == null) { + throw new ConversionException("Group column '" + column + "' not found in aggregation result columns"); + } + keyIndices.add(idx); + } + + Map, List> grouped = new LinkedHashMap<>(); + for (Object[] row : rows) { + List key = new ArrayList<>(keyIndices.size()); + for (int idx : keyIndices) { + key.add(row[idx]); + } + grouped.computeIfAbsent(key, k -> new ArrayList<>()).add(row); + } + return grouped; + } + + /** + * Computes the granularity key for an execution result from its plan's aggregate. + * The aggregate is found by walking down the plan's single-input spine — post-aggregate + * nodes (bucket-order sort, future HAVING filters) sit on top of it. + */ + private static String computeGranularityKey(ExecutionResult result) { + RelNode node = result.getPlan().relNode(); + while (node != null && !(node instanceof LogicalAggregate)) { + node = node.getInputs().isEmpty() ? null : node.getInput(0); + } + if (node instanceof LogicalAggregate agg) { + // Resolve group field names against the aggregate's INPUT row type via the group + // set — robust against output-side renames, unlike the aggregate's own row type. + List inputFields = agg.getInput().getRowType().getFieldNames(); + return granularityKey(agg.getGroupSet().asList().stream().map(inputFields::get).collect(Collectors.toList())); + } + return NO_GROUPING_KEY; + } + + /** + * Canonical granularity key: sorted, comma-joined group field names. Sorted because the + * plan emits group columns in schema order while the request walk accumulates them in + * nesting order — the two must produce the same key. + */ + private static String granularityKey(List groupFieldNames) { + return groupFieldNames.stream().sorted().collect(Collectors.joining(AGGREGATION_LEVEL_SEPARATOR)); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/SearchResponseBuilder.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/SearchResponseBuilder.java index 0c349b8056160..9e554b3a8435b 100644 --- a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/SearchResponseBuilder.java +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/result/SearchResponseBuilder.java @@ -8,38 +8,75 @@ package org.opensearch.dsl.result; +import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchResponseSections; import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.dsl.aggregation.AggregationRegistry; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.dsl.executor.QueryPlans; import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.InternalAggregations; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * Builds a {@link SearchResponse} from execution results. + * Handles conversion of flat execution results into OpenSearch response format. */ public class SearchResponseBuilder { private SearchResponseBuilder() {} /** - * Builds a SearchResponse from the given results and timing. + * Builds a SearchResponse from execution results. * - * @param results execution results from the plan executor - * @param convertTimeNanos time spent in DSL-to-RelNode conversion, in nanoseconds + * @param results execution results from the query executor + * @param request the original search request + * @param registry aggregation registry for building aggregations + * @param tookInMillis total execution time in milliseconds * @return a SearchResponse */ - // TODO: Analytics plugin should return execution metadata alongside Iterable rows: - // - executionTimeNanos: query execution time - // - totalDocCount: total matching documents for hits.total - // - terminatedEarly: whether execution was terminated early - // - timedOut: whether execution timed out - // - shardInfo: total/successful/skipped/failed shard counts - public static SearchResponse build(List results, long convertTimeNanos) { - // TODO: populate HITS and AGGREGATION plan types from results - long tookInMillis = convertTimeNanos / 1_000_000; - SearchHits hits = SearchHits.empty(true); - SearchResponseSections sections = new SearchResponseSections(hits, null, null, false, null, null, 0); - return new SearchResponse(sections, null, 0, 0, 0, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + public static SearchResponse build( + List results, + SearchRequest request, + AggregationRegistry registry, + long tookInMillis + ) throws ConversionException { + + SearchHits hits = buildHits(results); + InternalAggregations aggregations = buildAggregations(results, request, registry); + + SearchResponseSections sections = new SearchResponseSections(hits, aggregations, null, false, null, null, 0); + + // TODO: shard counts, timed_out, and engine-side took require execution metadata from + // the analytics plugin (returned alongside rows). Until then report a constant 1/1 — + // the analytics path has no per-shard fan-out to report. + return new SearchResponse(sections, null, 1, 1, 0, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + } + + private static SearchHits buildHits(List results) { + // TODO: Build hits from HITS results + return SearchHits.empty(true); + } + + private static InternalAggregations buildAggregations( + List results, + SearchRequest request, + AggregationRegistry registry + ) throws ConversionException { + + List aggResults = results.stream() + .filter(r -> r.getType() == QueryPlans.Type.AGGREGATION) + .collect(Collectors.toList()); + + if (aggResults.isEmpty() || request.source() == null || request.source().aggregations() == null) { + return null; + } + + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, aggResults); + return builder.build(new ArrayList<>(request.source().aggregations().getAggregatorFactories())); } } diff --git a/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/util/ComparisonUtils.java b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/util/ComparisonUtils.java new file mode 100644 index 0000000000000..559ca493ddaeb --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/main/java/org/opensearch/dsl/util/ComparisonUtils.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.util; + +import java.util.Objects; + +/** + * Utility methods for comparing values. + */ +public final class ComparisonUtils { + + private ComparisonUtils() {} + + /** + * Compares two values for equality, coercing numeric types (the engine may box the same + * column differently across plans). Integral pairs compare as long to avoid precision + * loss; floating-point pairs via {@link Double#compare}. No cross-type string coercion — + * bucket keys are typed by the schema. + */ + public static boolean valuesEqual(Object a, Object b) { + if (Objects.equals(a, b)) return true; + + if (a instanceof Number na && b instanceof Number nb) { + if (isIntegral(na) && isIntegral(nb)) { + return na.longValue() == nb.longValue(); + } + return Double.compare(na.doubleValue(), nb.doubleValue()) == 0; + } + + return false; + } + + private static boolean isIntegral(Number n) { + return n instanceof Long || n instanceof Integer || n instanceof Short || n instanceof Byte; + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilderTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilderTests.java index 2cf7bc4b9d591..daea761b950d0 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilderTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/AggregationMetadataBuilderTests.java @@ -51,4 +51,39 @@ public void testThrowsForUnknownField() { expectThrows(ConversionException.class, () -> builder.build(ctx.getRowType(), ctx.getCluster().getTypeFactory())); } + + public void testLiteralColumnAllocatorAssignsSequentialPositionsAfterBase() { + AggregationMetadataBuilder builder = new AggregationMetadataBuilder(); + LiteralColumnAllocator allocator = builder.literalColumnAllocator(4); + + assertEquals(4, allocator.columnFor(50.0)); + assertEquals(5, allocator.columnFor(95.0)); + assertEquals(6, allocator.integerColumnFor(200)); + assertEquals(7, allocator.coalescedColumnFor(1, 0.0)); + } + + public void testLiteralColumnAllocatorDeduplicatesEqualColumns() { + AggregationMetadataBuilder builder = new AggregationMetadataBuilder(); + LiteralColumnAllocator allocator = builder.literalColumnAllocator(4); + + assertEquals(4, allocator.columnFor(50.0)); + assertEquals(4, allocator.columnFor(50.0)); + assertEquals(5, allocator.coalescedColumnFor(1, 0.0)); + assertEquals(5, allocator.coalescedColumnFor(1, 0.0)); + // Same value, different kind or different field: distinct columns. + assertEquals(6, allocator.integerColumnFor(50)); + assertEquals(7, allocator.coalescedColumnFor(2, 0.0)); + } + + public void testLiteralColumnsTravelIntoMetadataInAllocationOrder() throws ConversionException { + AggregationMetadataBuilder builder = new AggregationMetadataBuilder(); + LiteralColumnAllocator allocator = builder.literalColumnAllocator(4); + allocator.coalescedColumnFor(1, 0.0); + allocator.columnFor(99.0); + builder.requestImplicitCount(); + + AggregationMetadata metadata = builder.build(ctx.getRowType(), ctx.getCluster().getTypeFactory()); + + assertEquals(List.of(LiteralColumn.coalesced(1, 0.0), LiteralColumn.constant(99.0)), metadata.getLiteralColumns()); + } } diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslatorTests.java index 21805e17eec64..f752d8f2be3b5 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslatorTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/bucket/TermsBucketTranslatorTests.java @@ -8,12 +8,19 @@ package org.opensearch.dsl.aggregation.bucket; +import org.opensearch.dsl.result.BucketEntry; import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.InternalOrder; +import org.opensearch.search.aggregations.bucket.terms.DoubleTerms; +import org.opensearch.search.aggregations.bucket.terms.LongTerms; +import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.test.OpenSearchTestCase; +import java.util.ArrayList; import java.util.List; public class TermsBucketTranslatorTests extends OpenSearchTestCase { @@ -103,7 +110,106 @@ public void testGetBucketOrderReturnsMetricOrder() { assertTrue(InternalOrder.isKeyAsc(compound.orderElements().get(1))); } - public void testToBucketAggregationNotYetImplemented() { - expectThrows(UnsupportedOperationException.class, () -> translator.toBucketAggregation(brandAgg, List.of())); + public void testToBucketAggregationBuildsStringTerms() { + List entries = List.of( + new BucketEntry(List.of("BrandA"), 3, InternalAggregations.EMPTY), + new BucketEntry(List.of("BrandB"), 2, InternalAggregations.EMPTY) + ); + + InternalAggregation agg = translator.toBucketAggregation(brandAgg, entries); + + assertTrue(agg instanceof StringTerms); + StringTerms terms = (StringTerms) agg; + assertEquals(brandAgg.getName(), terms.getName()); + assertEquals(2, terms.getBuckets().size()); + assertEquals("BrandA", terms.getBuckets().get(0).getKeyAsString()); + assertEquals(3, terms.getBuckets().get(0).getDocCount()); + assertEquals("BrandB", terms.getBuckets().get(1).getKeyAsString()); + assertEquals(2, terms.getBuckets().get(1).getDocCount()); + } + + /** Legacy parity: docs with a missing field form no bucket — a SQL NULL group is dropped. */ + public void testNullKeyBucketExcluded() { + List entries = new ArrayList<>(); + entries.add(new BucketEntry(java.util.Collections.singletonList(null), 4L, InternalAggregations.EMPTY)); + entries.add(new BucketEntry(List.of("BrandA"), 3L, InternalAggregations.EMPTY)); + + StringTerms terms = (StringTerms) translator.toBucketAggregation(brandAgg, entries); + + assertEquals(1, terms.getBuckets().size()); + assertEquals("BrandA", terms.getBuckets().get(0).getKeyAsString()); + } + + public void testToBucketAggregationEmptyBuckets() { + InternalAggregation agg = translator.toBucketAggregation(brandAgg, List.of()); + assertTrue(agg instanceof StringTerms); + assertTrue(((StringTerms) agg).getBuckets().isEmpty()); + } + + public void testIntegralKeysProduceLongTermsWithNumericKeys() { + TermsAggregationBuilder priceAgg = new TermsAggregationBuilder("by_price").field("price"); + List entries = List.of( + new BucketEntry(List.of(42L), 3, InternalAggregations.EMPTY), + new BucketEntry(List.of(7), 2, InternalAggregations.EMPTY) + ); + + InternalAggregation agg = translator.toBucketAggregation(priceAgg, entries); + + assertTrue(agg instanceof LongTerms); + LongTerms terms = (LongTerms) agg; + assertEquals(42L, terms.getBuckets().get(0).getKey()); + assertEquals("42", terms.getBuckets().get(0).getKeyAsString()); + assertEquals(7L, terms.getBuckets().get(1).getKey()); + } + + public void testFloatingKeysProduceDoubleTerms() { + TermsAggregationBuilder ratingAgg = new TermsAggregationBuilder("by_rating").field("rating"); + List entries = List.of(new BucketEntry(List.of(1.5), 3, InternalAggregations.EMPTY)); + + InternalAggregation agg = translator.toBucketAggregation(ratingAgg, entries); + + assertTrue(agg instanceof DoubleTerms); + assertEquals(1.5, ((DoubleTerms) agg).getBuckets().get(0).getKey()); + } + + public void testBooleanKeysRenderLikeClassicBooleanTerms() { + TermsAggregationBuilder boolAgg = new TermsAggregationBuilder("by_flag").field("flag"); + List entries = List.of( + new BucketEntry(List.of(true), 3, InternalAggregations.EMPTY), + new BucketEntry(List.of(false), 2, InternalAggregations.EMPTY) + ); + + LongTerms terms = (LongTerms) translator.toBucketAggregation(boolAgg, entries); + + assertEquals(1L, terms.getBuckets().get(0).getKey()); + assertEquals("true", terms.getBuckets().get(0).getKeyAsString()); + assertEquals(0L, terms.getBuckets().get(1).getKey()); + assertEquals("false", terms.getBuckets().get(1).getKeyAsString()); + } + + public void testBinaryKeysDecodeToIpAddressStrings() { + TermsAggregationBuilder ipAgg = new TermsAggregationBuilder("by_ip").field("ip"); + List entries = List.of( + new BucketEntry(List.of(new byte[] { 10, 0, 0, 1 }), 3, InternalAggregations.EMPTY), + new BucketEntry( + List.of(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xff, (byte) 0xff, 10, 0, 0, 2 }), + 2, + InternalAggregations.EMPTY + ) + ); + + StringTerms terms = (StringTerms) translator.toBucketAggregation(ipAgg, entries); + + assertEquals("10.0.0.1", terms.getBuckets().get(0).getKeyAsString()); + assertEquals("10.0.0.2", terms.getBuckets().get(1).getKeyAsString()); + } + + public void testUndecodableBinaryKeyFallsBackToBase64() { + TermsAggregationBuilder ipAgg = new TermsAggregationBuilder("by_ip").field("ip"); + List entries = List.of(new BucketEntry(List.of(new byte[] { 1, 2, 3 }), 1, InternalAggregations.EMPTY)); + + StringTerms terms = (StringTerms) translator.toBucketAggregation(ipAgg, entries); + + assertEquals("AQID", terms.getBuckets().get(0).getKeyAsString()); } } diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslatorTests.java new file mode 100644 index 0000000000000..65974dd9f72d7 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ExtendedStatsMetricTranslatorTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.aggregation.AggregationMetadataBuilder; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.InternalExtendedStats; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +public class ExtendedStatsMetricTranslatorTests extends OpenSearchTestCase { + + private final ConversionContext ctx = TestUtils.createContext(); + private final ExtendedStatsMetricTranslator translator = new ExtendedStatsMetricTranslator(); + + public void testGetAggregationType() { + assertEquals(ExtendedStatsAggregationBuilder.class, translator.getAggregationType()); + } + + public void testToAggregateCalls() throws ConversionException { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType()); + + assertEquals(5, calls.size()); + assertEquals(SqlKind.COUNT, calls.get(0).getAggregation().getKind()); + assertEquals(SqlKind.MIN, calls.get(1).getAggregation().getKind()); + assertEquals(SqlKind.MAX, calls.get(2).getAggregation().getKind()); + assertEquals(SqlKind.SUM, calls.get(3).getAggregation().getKind()); + assertEquals("price_stats_variance", calls.get(4).getName()); + } + + public void testToAggregateCallsInvalidField() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("invalid"); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + } + + public void testNonNumericFieldRejected() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("brand_stats").field("brand"); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + + assertEquals("Field [brand] of type [VARCHAR] is not supported for aggregation [extended_stats]", e.getMessage()); + } + + public void testMissingAggregatesAllCallsOverCoalescedColumn() throws ConversionException { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + agg.missing(0); + int baseFieldCount = ctx.getRowType().getFieldCount(); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(baseFieldCount); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator); + + assertEquals(5, calls.size()); + for (AggregateCall call : calls) { + assertEquals(List.of(baseFieldCount), call.getArgList()); + } + } + + public void testGetAggregateFieldNames() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + + List names = translator.getAggregateFieldNames(agg); + + assertEquals(List.of("price_stats_count", "price_stats_min", "price_stats_max", "price_stats_sum", "price_stats_variance"), names); + } + + public void testToInternalAggregationWithValidValues() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + Map values = Map.of( + "price_stats_count", + 10L, + "price_stats_min", + 5.0, + "price_stats_max", + 100.0, + "price_stats_sum", + 550.0, + "price_stats_variance", + 900.0 + ); + + InternalAggregation result = translator.toInternalAggregation(agg, values); + + InternalExtendedStats stats = (InternalExtendedStats) result; + assertEquals(10, stats.getCount()); + assertEquals(5.0, stats.getMin(), 0.001); + assertEquals(100.0, stats.getMax(), 0.001); + assertEquals(550.0, stats.getSum(), 0.001); + // sumOfSquares = (variance + avg^2) * count = (900 + 55^2) * 10 + assertEquals(39250.0, stats.getSumOfSquares(), 0.001); + assertEquals(900.0, stats.getVariance(), 0.001); + assertEquals(30.0, stats.getStdDeviation(), 0.001); + } + + public void testSigmaFromRequestShapesBounds() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price").sigma(3.0); + Map values = Map.of( + "price_stats_count", + 10L, + "price_stats_min", + 5.0, + "price_stats_max", + 100.0, + "price_stats_sum", + 550.0, + "price_stats_variance", + 900.0 + ); + + InternalExtendedStats stats = (InternalExtendedStats) translator.toInternalAggregation(agg, values); + + assertEquals(3.0, stats.getSigma(), 0.001); + // upper bound = avg + sigma * stddev = 55 + 3 * 30 + assertEquals(145.0, stats.getStdDeviationBound(InternalExtendedStats.Bounds.UPPER), 0.001); + } + + public void testToInternalAggregationWithNull() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, null); + + InternalExtendedStats stats = (InternalExtendedStats) result; + assertEquals(0, stats.getCount()); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0.001); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0.001); + assertEquals(0.0, stats.getSum(), 0.001); + assertEquals(0.0, stats.getSumOfSquares(), 0.001); + assertTrue(Double.isNaN(stats.getVariance())); + } + + public void testToInternalAggregationWithPartialNulls() { + ExtendedStatsAggregationBuilder agg = new ExtendedStatsAggregationBuilder("price_stats").field("price"); + Map values = Map.of("price_stats_count", 5L, "price_stats_sum", 100.0); + + InternalAggregation result = translator.toInternalAggregation(agg, values); + + InternalExtendedStats stats = (InternalExtendedStats) result; + assertEquals(5, stats.getCount()); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0.001); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0.001); + assertEquals(100.0, stats.getSum(), 0.001); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentilesTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentilesTests.java new file mode 100644 index 0000000000000..0bad7aa859326 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/InternalDslPercentilesTests.java @@ -0,0 +1,177 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.DocValueFormat; +import org.opensearch.search.aggregations.metrics.Percentile; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class InternalDslPercentilesTests extends OpenSearchTestCase { + + public void testKeyedRendering() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0, 99.0 }, + new double[] { 899.0, 1299.0 }, + true, + DocValueFormat.RAW + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals("{\"values\":{\"50.0\":899.0,\"99.0\":1299.0}}", builder.toString()); + } + } + + public void testKeyedRenderingWithMissingValue() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0, 99.0 }, + new double[] { 899.0, Double.NaN }, + true, + DocValueFormat.RAW + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals("{\"values\":{\"50.0\":899.0,\"99.0\":null}}", builder.toString()); + } + } + + public void testNonKeyedRendering() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0 }, + new double[] { 899.0 }, + false, + DocValueFormat.RAW + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals("{\"values\":[{\"key\":50.0,\"value\":899.0}]}", builder.toString()); + } + } + + public void testKeyedRenderingWithFormat() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0, 99.0 }, + new double[] { 899.0, 1299.5 }, + true, + new DocValueFormat.Decimal("0.00") + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals( + "{\"values\":{\"50.0\":899.0,\"50.0_as_string\":\"899.00\",\"99.0\":1299.5,\"99.0_as_string\":\"1299.50\"}}", + builder.toString() + ); + } + } + + public void testNonKeyedRenderingWithFormat() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0 }, + new double[] { 899.0 }, + false, + new DocValueFormat.Decimal("0.00") + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals("{\"values\":[{\"key\":50.0,\"value\":899.0,\"value_as_string\":\"899.00\"}]}", builder.toString()); + } + } + + public void testFormatSkipsAsStringForEmptyValue() throws Exception { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0 }, + new double[] { Double.NaN }, + true, + new DocValueFormat.Decimal("0.00") + ); + + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + agg.doXContentBody(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals("{\"values\":{\"50.0\":null}}", builder.toString()); + } + } + + public void testPercentileAsStringUsesFormat() { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0 }, + new double[] { 899.0 }, + true, + new DocValueFormat.Decimal("0.00") + ); + assertEquals("899.00", agg.percentileAsString(50.0)); + } + + public void testTypeNameMatchesLegacyTDigest() { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0 }, + new double[] { 1.0 }, + true, + DocValueFormat.RAW + ); + assertEquals("tdigest_percentiles", agg.getWriteableName()); + assertEquals("tdigest_percentiles", agg.getType()); + } + + public void testAccessors() { + InternalDslPercentiles agg = new InternalDslPercentiles( + "pcts", + new double[] { 50.0, 99.0 }, + new double[] { 899.0, 1299.0 }, + true, + DocValueFormat.RAW + ); + + assertEquals(899.0, agg.percentile(50.0), 0.001); + assertEquals(899.0, agg.value("50.0"), 0.001); + expectThrows(IllegalArgumentException.class, () -> agg.percentile(75.0)); + + List entries = new ArrayList<>(); + agg.iterator().forEachRemaining(entries::add); + assertEquals(2, entries.size()); + assertEquals(50.0, entries.get(0).getPercent(), 0.001); + assertEquals(1299.0, entries.get(1).getValue(), 0.001); + } + + public void testLengthMismatchRejected() { + expectThrows( + IllegalArgumentException.class, + () -> new InternalDslPercentiles("pcts", new double[] { 50.0, 99.0 }, new double[] { 1.0 }, true, DocValueFormat.RAW) + ); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/MetricTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/MetricTranslatorTests.java index af3e51d39a3f4..77f4b026217cd 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/MetricTranslatorTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/MetricTranslatorTests.java @@ -11,22 +11,30 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.sql.SqlKind; import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.aggregation.AggregationMetadataBuilder; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; import org.opensearch.dsl.converter.ConversionContext; import org.opensearch.dsl.converter.ConversionException; import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import org.opensearch.search.aggregations.metrics.MinAggregationBuilder; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.test.OpenSearchTestCase; +import java.util.List; +import java.util.Map; + public class MetricTranslatorTests extends OpenSearchTestCase { private final ConversionContext ctx = TestUtils.createContext(); public void testAvgTranslator() throws ConversionException { AvgMetricTranslator translator = new AvgMetricTranslator(); - AggregateCall call = translator.toAggregateCall(new AvgAggregationBuilder("avg_price").field("price"), ctx.getRowType()); + List calls = translator.toAggregateCalls(new AvgAggregationBuilder("avg_price").field("price"), ctx.getRowType()); + assertEquals(1, calls.size()); + AggregateCall call = calls.get(0); assertEquals(SqlKind.AVG, call.getAggregation().getKind()); assertEquals("avg_price", call.getName()); assertEquals(1, call.getArgList().size()); @@ -35,24 +43,30 @@ public void testAvgTranslator() throws ConversionException { public void testSumTranslator() throws ConversionException { SumMetricTranslator translator = new SumMetricTranslator(); - AggregateCall call = translator.toAggregateCall(new SumAggregationBuilder("total").field("price"), ctx.getRowType()); + List calls = translator.toAggregateCalls(new SumAggregationBuilder("total").field("price"), ctx.getRowType()); + assertEquals(1, calls.size()); + AggregateCall call = calls.get(0); assertEquals(SqlKind.SUM, call.getAggregation().getKind()); assertEquals("total", call.getName()); } public void testMinTranslator() throws ConversionException { MinMetricTranslator translator = new MinMetricTranslator(); - AggregateCall call = translator.toAggregateCall(new MinAggregationBuilder("min_price").field("price"), ctx.getRowType()); + List calls = translator.toAggregateCalls(new MinAggregationBuilder("min_price").field("price"), ctx.getRowType()); + assertEquals(1, calls.size()); + AggregateCall call = calls.get(0); assertEquals(SqlKind.MIN, call.getAggregation().getKind()); assertEquals("min_price", call.getName()); } public void testMaxTranslator() throws ConversionException { MaxMetricTranslator translator = new MaxMetricTranslator(); - AggregateCall call = translator.toAggregateCall(new MaxAggregationBuilder("max_price").field("price"), ctx.getRowType()); + List calls = translator.toAggregateCalls(new MaxAggregationBuilder("max_price").field("price"), ctx.getRowType()); + assertEquals(1, calls.size()); + AggregateCall call = calls.get(0); assertEquals(SqlKind.MAX, call.getAggregation().getKind()); assertEquals("max_price", call.getName()); } @@ -62,12 +76,88 @@ public void testThrowsForUnknownField() { expectThrows( ConversionException.class, - () -> translator.toAggregateCall(new AvgAggregationBuilder("bad").field("nonexistent"), ctx.getRowType()) + () -> translator.toAggregateCalls(new AvgAggregationBuilder("bad").field("nonexistent"), ctx.getRowType()) ); } + public void testNonNumericFieldRejectedWithClassicMessage() { + AvgMetricTranslator translator = new AvgMetricTranslator(); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> translator.toAggregateCalls(new AvgAggregationBuilder("bad").field("brand"), ctx.getRowType()) + ); + + assertEquals("Field [brand] of type [VARCHAR] is not supported for aggregation [avg]", e.getMessage()); + } + + public void testNonNumericFieldRejectedForAllArithmeticMetrics() { + assertNonNumericRejected(new SumMetricTranslator(), new SumAggregationBuilder("s").field("brand"), "sum"); + assertNonNumericRejected(new MinMetricTranslator(), new MinAggregationBuilder("m").field("brand"), "min"); + assertNonNumericRejected(new MaxMetricTranslator(), new MaxAggregationBuilder("x").field("brand"), "max"); + } + + private void assertNonNumericRejected( + MetricTranslator translator, + T agg, + String aggregationType + ) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + assertTrue(e.getMessage().contains("is not supported for aggregation [" + aggregationType + "]")); + } + public void testAggregateFieldName() { AvgMetricTranslator translator = new AvgMetricTranslator(); - assertEquals("avg_price", translator.getAggregateFieldName(new AvgAggregationBuilder("avg_price").field("price"))); + List names = translator.getAggregateFieldNames(new AvgAggregationBuilder("avg_price").field("price")); + assertEquals(1, names.size()); + assertEquals("avg_price", names.get(0)); + } + + public void testMissingAggregatesOverCoalescedColumn() throws ConversionException { + AvgMetricTranslator translator = new AvgMetricTranslator(); + AvgAggregationBuilder agg = new AvgAggregationBuilder("avg_price").field("price"); + agg.missing(0); + int baseFieldCount = ctx.getRowType().getFieldCount(); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(baseFieldCount); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator); + + assertEquals(List.of(baseFieldCount), calls.get(0).getArgList()); + } + + public void testMissingWithoutAllocatorRejected() { + AvgMetricTranslator translator = new AvgMetricTranslator(); + AvgAggregationBuilder agg = new AvgAggregationBuilder("avg_price").field("price"); + agg.missing(0); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + } + + public void testNonNumericMissingRejected() { + AvgMetricTranslator translator = new AvgMetricTranslator(); + AvgAggregationBuilder agg = new AvgAggregationBuilder("avg_price").field("price"); + agg.missing("not-a-number"); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(ctx.getRowType().getFieldCount()); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator)); + } + + public void testFormatAppliedToResponse() { + AvgMetricTranslator translator = new AvgMetricTranslator(); + AvgAggregationBuilder agg = new AvgAggregationBuilder("avg_price").field("price"); + agg.format("0.00"); + + InternalAvg result = (InternalAvg) translator.toInternalAggregation(agg, Map.of("avg_price", 30.2)); + + assertEquals("30.20", result.getValueAsString()); + } + + public void testInvalidFormatRejected() { + AvgMetricTranslator translator = new AvgMetricTranslator(); + AvgAggregationBuilder agg = new AvgAggregationBuilder("avg_price").field("price"); + agg.format("0.0.0"); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(ctx.getRowType().getFieldCount()); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator)); } } diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslatorTests.java new file mode 100644 index 0000000000000..7a86ad7630084 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/PercentilesMetricTranslatorTests.java @@ -0,0 +1,242 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.core.AggregateCall; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.aggregation.AggregationMetadataBuilder; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; +import org.opensearch.search.aggregations.metrics.PercentilesConfig; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +public class PercentilesMetricTranslatorTests extends OpenSearchTestCase { + + private final ConversionContext ctx = TestUtils.createContext(); + private final PercentilesMetricTranslator translator = new PercentilesMetricTranslator(); + + private LiteralColumnAllocator allocator() { + return new AggregationMetadataBuilder().literalColumnAllocator(ctx.getRowType().getFieldCount()); + } + + public void testGetAggregationType() { + assertEquals(PercentilesAggregationBuilder.class, translator.getAggregationType()); + } + + public void testToAggregateCallsOnePerPercent() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 95, 99); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + assertEquals(3, calls.size()); + for (AggregateCall call : calls) { + assertEquals("PERCENTILE_APPROX", call.getAggregation().getName()); + assertEquals(2, call.getArgList().size()); + } + assertEquals("pcts_p50_0", calls.get(0).getName()); + assertEquals("pcts_p95_0", calls.get(1).getName()); + assertEquals("pcts_p99_0", calls.get(2).getName()); + // Literal columns are appended after the base fields, one per distinct percent. + assertEquals(List.of(1, baseFieldCount), calls.get(0).getArgList()); + assertEquals(List.of(1, baseFieldCount + 1), calls.get(1).getArgList()); + assertEquals(List.of(1, baseFieldCount + 2), calls.get(2).getArgList()); + } + + public void testLiteralColumnsDeduplicateAcrossAggregations() throws ConversionException { + LiteralColumnAllocator shared = allocator(); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + PercentilesAggregationBuilder first = new PercentilesAggregationBuilder("a").field("price").percentiles(50, 95); + PercentilesAggregationBuilder second = new PercentilesAggregationBuilder("b").field("rating").percentiles(95, 99); + + List firstCalls = translator.toAggregateCalls(first, ctx.getRowType(), shared); + List secondCalls = translator.toAggregateCalls(second, ctx.getRowType(), shared); + + // 95.0 is shared: 50.0→base, 95.0→base+1, 99.0→base+2. + assertEquals(List.of(1, baseFieldCount), firstCalls.get(0).getArgList()); + assertEquals(List.of(1, baseFieldCount + 1), firstCalls.get(1).getArgList()); + assertEquals(List.of(3, baseFieldCount + 1), secondCalls.get(0).getArgList()); + assertEquals(List.of(3, baseFieldCount + 2), secondCalls.get(1).getArgList()); + } + + public void testToAggregateCallsInvalidField() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("invalid").percentiles(50); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator())); + } + + public void testNonNumericFieldRejectedWithClassicMessage() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("brand").percentiles(50); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator()) + ); + + assertEquals("Field [brand] of type [VARCHAR] is not supported for aggregation [percentiles]", e.getMessage()); + } + + public void testTwoArgVariantRejected() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + } + + public void testHdrMethodRejected() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price") + .percentiles(50) + .percentilesConfig(new PercentilesConfig.Hdr()); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator())); + } + + public void testGetAggregateFieldNames() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 99.9); + + assertEquals(List.of("pcts_p50_0", "pcts_p99_9"), translator.getAggregateFieldNames(agg)); + } + + public void testToInternalAggregationWithValues() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 99); + Map values = Map.of("pcts_p50_0", 899, "pcts_p99_0", 1299.0); + + InternalDslPercentiles result = (InternalDslPercentiles) translator.toInternalAggregation(agg, values); + + assertEquals(899.0, result.percentile(50), 0.001); + assertEquals(1299.0, result.percentile(99), 0.001); + } + + public void testToInternalAggregationWithNull() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 99); + + InternalDslPercentiles result = (InternalDslPercentiles) translator.toInternalAggregation(agg, null); + + assertTrue(Double.isNaN(result.percentile(50))); + assertTrue(Double.isNaN(result.percentile(99))); + } + + public void testToInternalAggregationWithPartialValues() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 99); + Map values = Map.of("pcts_p50_0", 899.0); + + InternalDslPercentiles result = (InternalDslPercentiles) translator.toInternalAggregation(agg, values); + + assertEquals(899.0, result.percentile(50), 0.001); + assertTrue(Double.isNaN(result.percentile(99))); + } + + public void testMissingAllocatesSharedCoalescedColumn() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 95); + agg.missing(0); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + // One coalesced column (allocated first) shared by all percents; percent literals follow. + assertEquals(2, calls.size()); + assertEquals(List.of(baseFieldCount, baseFieldCount + 1), calls.get(0).getArgList()); + assertEquals(List.of(baseFieldCount, baseFieldCount + 2), calls.get(1).getArgList()); + } + + public void testStringMissingValueParsed() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.missing("2.5"); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + assertEquals(List.of(baseFieldCount, baseFieldCount + 1), calls.get(0).getArgList()); + } + + public void testNonNumericMissingRejected() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.missing("not-a-number"); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator())); + } + + public void testInvalidFormatRejected() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.format("0.0.0"); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator())); + } + + public void testFormatAppliedToResponse() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.format("0.00"); + Map values = Map.of("pcts_p50_0", 899.0); + + InternalDslPercentiles result = (InternalDslPercentiles) translator.toInternalAggregation(agg, values); + + assertEquals("899.00", result.percentileAsString(50.0)); + } + + public void testExplicitCompressionEmitsThreeArgForm() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50, 95); + agg.percentilesConfig(new PercentilesConfig.TDigest(200)); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + // One centroids column (allocated first) shared by all percents; percent literals follow. + assertEquals(2, calls.size()); + for (AggregateCall call : calls) { + assertEquals("PERCENTILE_APPROX_N", call.getAggregation().getName()); + } + assertEquals(List.of(1, baseFieldCount + 1, baseFieldCount), calls.get(0).getArgList()); + assertEquals(List.of(1, baseFieldCount + 2, baseFieldCount), calls.get(1).getArgList()); + } + + public void testDefaultConfigKeepsTwoArgForm() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + assertEquals("PERCENTILE_APPROX", calls.get(0).getAggregation().getName()); + assertEquals(2, calls.get(0).getArgList().size()); + } + + public void testDefaultCompressionValueKeepsTwoArgForm() throws ConversionException { + // The request parser injects TDigest(100) when the JSON names no method; must equal no-config. + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.percentilesConfig(new PercentilesConfig.TDigest(100)); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + assertEquals("PERCENTILE_APPROX", calls.get(0).getAggregation().getName()); + assertEquals(2, calls.get(0).getArgList().size()); + } + + public void testZeroCompressionRejected() { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.percentilesConfig(new PercentilesConfig.TDigest(0)); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator())); + } + + public void testCompressionCombinesWithMissing() throws ConversionException { + PercentilesAggregationBuilder agg = new PercentilesAggregationBuilder("pcts").field("price").percentiles(50); + agg.missing(0); + agg.percentilesConfig(new PercentilesConfig.TDigest(150)); + int baseFieldCount = ctx.getRowType().getFieldCount(); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator()); + + // Allocation order: coalesced field, centroids, percent. + assertEquals("PERCENTILE_APPROX_N", calls.get(0).getAggregation().getName()); + assertEquals(List.of(baseFieldCount, baseFieldCount + 2, baseFieldCount + 1), calls.get(0).getArgList()); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslatorTests.java new file mode 100644 index 0000000000000..557760a8b89ed --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/StatsMetricTranslatorTests.java @@ -0,0 +1,150 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.aggregation.AggregationMetadataBuilder; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalStats; +import org.opensearch.search.aggregations.metrics.StatsAggregationBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +public class StatsMetricTranslatorTests extends OpenSearchTestCase { + + private final ConversionContext ctx = TestUtils.createContext(); + private final StatsMetricTranslator translator = new StatsMetricTranslator(); + + public void testGetAggregationType() { + assertEquals(StatsAggregationBuilder.class, translator.getAggregationType()); + } + + public void testToAggregateCalls() throws ConversionException { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType()); + + assertEquals(4, calls.size()); + assertEquals(SqlKind.COUNT, calls.get(0).getAggregation().getKind()); + assertEquals(SqlKind.MIN, calls.get(1).getAggregation().getKind()); + assertEquals(SqlKind.MAX, calls.get(2).getAggregation().getKind()); + assertEquals(SqlKind.SUM, calls.get(3).getAggregation().getKind()); + assertEquals("price_stats_count", calls.get(0).getName()); + } + + public void testToAggregateCallsInvalidField() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("invalid"); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + } + + public void testNonNumericFieldRejected() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("brand_stats").field("brand"); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + + assertEquals("Field [brand] of type [VARCHAR] is not supported for aggregation [stats]", e.getMessage()); + } + + public void testMissingAggregatesAllCallsOverCoalescedColumn() throws ConversionException { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + agg.missing(0); + int baseFieldCount = ctx.getRowType().getFieldCount(); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(baseFieldCount); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator); + + assertEquals(4, calls.size()); + for (AggregateCall call : calls) { + assertEquals(List.of(baseFieldCount), call.getArgList()); + } + } + + public void testFormatAppliedToResponse() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + agg.format("0.00"); + Map values = Map.of( + "price_stats_count", + 2L, + "price_stats_min", + 1.0, + "price_stats_max", + 3.0, + "price_stats_sum", + 4.0 + ); + + InternalStats result = (InternalStats) translator.toInternalAggregation(agg, values); + + assertEquals("2.00", result.getAvgAsString()); + } + + public void testGetAggregateFieldNames() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + + List names = translator.getAggregateFieldNames(agg); + + assertEquals(List.of("price_stats_count", "price_stats_min", "price_stats_max", "price_stats_sum"), names); + } + + public void testToInternalAggregationWithValidValues() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + Map values = Map.of( + "price_stats_count", + 10L, + "price_stats_min", + 5.0, + "price_stats_max", + 100.0, + "price_stats_sum", + 550.0 + ); + + InternalAggregation result = translator.toInternalAggregation(agg, values); + + InternalStats stats = (InternalStats) result; + assertEquals(10, stats.getCount()); + assertEquals(5.0, stats.getMin(), 0.001); + assertEquals(100.0, stats.getMax(), 0.001); + assertEquals(550.0, stats.getSum(), 0.001); + assertEquals(55.0, stats.getAvg(), 0.001); + } + + public void testToInternalAggregationWithNull() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, null); + + InternalStats stats = (InternalStats) result; + assertEquals(0, stats.getCount()); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0.001); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0.001); + assertEquals(0.0, stats.getSum(), 0.001); + } + + public void testToInternalAggregationWithPartialNulls() { + StatsAggregationBuilder agg = new StatsAggregationBuilder("price_stats").field("price"); + Map values = Map.of("price_stats_count", 5L, "price_stats_sum", 100.0); + + InternalAggregation result = translator.toInternalAggregation(agg, values); + + InternalStats stats = (InternalStats) result; + assertEquals(5, stats.getCount()); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0.001); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0.001); + assertEquals(100.0, stats.getSum(), 0.001); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslatorTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslatorTests.java new file mode 100644 index 0000000000000..1a938c2223ac9 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/aggregation/metric/ValueCountMetricTranslatorTests.java @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.aggregation.metric; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.sql.SqlKind; +import org.opensearch.dsl.TestUtils; +import org.opensearch.dsl.aggregation.AggregationMetadataBuilder; +import org.opensearch.dsl.aggregation.LiteralColumnAllocator; +import org.opensearch.dsl.converter.ConversionContext; +import org.opensearch.dsl.converter.ConversionException; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.InternalValueCount; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +public class ValueCountMetricTranslatorTests extends OpenSearchTestCase { + + private final ConversionContext ctx = TestUtils.createContext(); + private final ValueCountMetricTranslator translator = new ValueCountMetricTranslator(); + + public void testGetAggregationType() { + assertEquals(ValueCountAggregationBuilder.class, translator.getAggregationType()); + } + + public void testToAggregateCallsReturnsCountFunction() throws ConversionException { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType()); + + assertEquals(1, calls.size()); + AggregateCall call = calls.get(0); + assertEquals(SqlKind.COUNT, call.getAggregation().getKind()); + assertEquals("price_count", call.getName()); + } + + public void testToAggregateCallsInvalidField() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("count").field("invalid"); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType())); + } + + /** Classic parity: value_count accepts any field type (it only tests value presence). */ + public void testNonNumericFieldAccepted() throws ConversionException { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("brand_count").field("brand"); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType()); + + assertEquals(1, calls.size()); + assertEquals(SqlKind.COUNT, calls.get(0).getAggregation().getKind()); + } + + public void testNumericMissingCountsOverCoalescedColumn() throws ConversionException { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + agg.missing(0); + int baseFieldCount = ctx.getRowType().getFieldCount(); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(baseFieldCount); + + List calls = translator.toAggregateCalls(agg, ctx.getRowType(), allocator); + + assertEquals(List.of(baseFieldCount), calls.get(0).getArgList()); + } + + /** Non-numeric substitutes are rejected — the substitute column is numeric on this path. */ + public void testNonNumericMissingRejected() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("brand_count").field("brand"); + agg.missing("N/A"); + LiteralColumnAllocator allocator = new AggregationMetadataBuilder().literalColumnAllocator(ctx.getRowType().getFieldCount()); + + expectThrows(ConversionException.class, () -> translator.toAggregateCalls(agg, ctx.getRowType(), allocator)); + } + + public void testGetAggregateFieldNames() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + List names = translator.getAggregateFieldNames(agg); + + assertEquals(List.of("price_count"), names); + } + + public void testToInternalAggregationWithValidValue() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, Map.of("price_count", 42L)); + + InternalValueCount count = (InternalValueCount) result; + assertEquals(42L, count.getValue()); + } + + public void testToInternalAggregationWithZero() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, Map.of("price_count", 0L)); + + InternalValueCount count = (InternalValueCount) result; + assertEquals(0L, count.getValue()); + } + + public void testToInternalAggregationWithNull() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, null); + + InternalValueCount count = (InternalValueCount) result; + assertEquals(0L, count.getValue()); + } + + public void testToInternalAggregationWithNumberType() { + ValueCountAggregationBuilder agg = new ValueCountAggregationBuilder("price_count").field("price"); + + InternalAggregation result = translator.toInternalAggregation(agg, Map.of("price_count", 100)); + + InternalValueCount count = (InternalValueCount) result; + assertEquals(100L, count.getValue()); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/AggregationResponseBuilderTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/AggregationResponseBuilderTests.java new file mode 100644 index 0000000000000..858f39f5976db --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/AggregationResponseBuilderTests.java @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.result; + +import org.opensearch.dsl.aggregation.AggregationRegistry; +import org.opensearch.dsl.aggregation.GroupingInfo; +import org.opensearch.dsl.aggregation.bucket.BucketTranslator; +import org.opensearch.dsl.aggregation.metric.MetricTranslator; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AggregationResponseBuilderTests extends OpenSearchTestCase { + + public void testBuildEmptyAggregations() throws Exception { + AggregationRegistry registry = new AggregationRegistry(); + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, List.of()); + + InternalAggregations aggs = builder.build(List.of()); + assertNotNull(aggs); + assertEquals(0, aggs.asList().size()); + } + + public void testBuildMetricWithNoResults() throws Exception { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(createMetricTranslator(AvgAggregationBuilder.class)); + + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, List.of()); + AvgAggregationBuilder avgAgg = new AvgAggregationBuilder("avg_price").field("price"); + + InternalAggregations aggs = builder.build(List.of(avgAgg)); + assertEquals(1, aggs.asList().size()); + assertEquals("avg_price", aggs.asList().get(0).getName()); + } + + public void testBuildBucketWithNoResults() throws Exception { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(createBucketTranslator(TermsAggregationBuilder.class, "brand")); + + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, List.of()); + TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("by_brand").field("brand"); + + InternalAggregations aggs = builder.build(List.of(termsAgg)); + assertEquals(1, aggs.asList().size()); + assertEquals("by_brand", aggs.asList().get(0).getName()); + } + + public void testBuildMultipleAggregations() throws Exception { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(createMetricTranslator(AvgAggregationBuilder.class)); + registry.register(createBucketTranslator(TermsAggregationBuilder.class, "brand")); + + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, List.of()); + + AvgAggregationBuilder avgAgg = new AvgAggregationBuilder("avg_price").field("price"); + TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("by_brand").field("brand"); + + InternalAggregations aggs = builder.build(List.of(avgAgg, termsAgg)); + assertEquals(2, aggs.asList().size()); + } + + public void testBuildNestedAggregation() throws Exception { + AggregationRegistry registry = new AggregationRegistry(); + registry.register(createBucketTranslator(TermsAggregationBuilder.class, "brand")); + registry.register(createMetricTranslator(AvgAggregationBuilder.class)); + + AggregationResponseBuilder builder = new AggregationResponseBuilder(registry, List.of()); + + TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("by_brand").field("brand") + .subAggregation(new AvgAggregationBuilder("avg_price").field("price")); + + InternalAggregations aggs = builder.build(List.of(termsAgg)); + assertEquals(1, aggs.asList().size()); + assertEquals("by_brand", aggs.asList().get(0).getName()); + } + + @SuppressWarnings("unchecked") + private MetricTranslator createMetricTranslator(Class aggClass) { + MetricTranslator translator = mock(MetricTranslator.class); + when(translator.getAggregationType()).thenReturn(aggClass); + when(translator.toInternalAggregation(any(), any())).thenAnswer(inv -> { + InternalAggregation agg = mock(InternalAggregation.class); + when(agg.getName()).thenReturn(((AggregationBuilder) inv.getArgument(0)).getName()); + return agg; + }); + return translator; + } + + @SuppressWarnings("unchecked") + private BucketTranslator createBucketTranslator(Class aggClass, String fieldName) { + BucketTranslator translator = mock(BucketTranslator.class); + when(translator.getAggregationType()).thenReturn(aggClass); + + GroupingInfo grouping = mock(GroupingInfo.class); + when(grouping.getFieldNames()).thenReturn(List.of(fieldName)); + when(translator.getGrouping(any())).thenReturn(grouping); + when(translator.getSubAggregations(any())).thenReturn(List.of()); + + when(translator.toBucketAggregation(any(), any())).thenAnswer(inv -> { + InternalAggregation agg = mock(InternalAggregation.class); + when(agg.getName()).thenReturn(((AggregationBuilder) inv.getArgument(0)).getName()); + return agg; + }); + return translator; + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java index 163ad3a570378..e3011dc4f5b3c 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/result/SearchResponseBuilderTests.java @@ -8,6 +8,7 @@ package org.opensearch.dsl.result; +import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; @@ -17,12 +18,16 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.dsl.aggregation.AggregationRegistry; import org.opensearch.dsl.converter.SearchSourceConverter; import org.opensearch.dsl.executor.QueryPlans; import org.opensearch.dsl.golden.CalciteTestInfra; import org.opensearch.dsl.golden.GoldenFileLoader; import org.opensearch.dsl.golden.GoldenTestCase; import org.opensearch.search.SearchModule; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.bucket.terms.StringTerms; +import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.OpenSearchTestCase; @@ -39,13 +44,161 @@ public class SearchResponseBuilderTests extends OpenSearchTestCase { - public void testBuildReturnsEmptyResponse() { - SearchResponse response = SearchResponseBuilder.build(List.of(), 42L * 1_000_000); + public void testBuildWithNoResults() throws Exception { + SearchRequest request = new SearchRequest(); + request.source(new SearchSourceBuilder()); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 42L); assertNotNull(response); assertEquals(200, response.status().getStatus()); assertEquals(0, response.getHits().getHits().length); assertEquals(42L, response.getTook().millis()); + assertNull(response.getAggregations()); + assertEquals(1, response.getTotalShards()); + assertEquals(1, response.getSuccessfulShards()); + } + + public void testBuildWithEmptyRequest() throws Exception { + SearchRequest request = new SearchRequest(); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 100L); + + assertNotNull(response); + assertEquals(200, response.status().getStatus()); + assertEquals(100L, response.getTook().millis()); + assertNull(response.getAggregations()); + } + + public void testBuildWithNullSource() throws Exception { + SearchRequest request = new SearchRequest(); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 50L); + + assertNotNull(response); + assertEquals(200, response.status().getStatus()); + assertNull(response.getAggregations()); + assertEquals(1, response.getTotalShards()); + } + + public void testBuildWithAggregationsButNoResults() throws Exception { + SearchRequest request = new SearchRequest(); + SearchSourceBuilder source = new SearchSourceBuilder(); + source.aggregation(AggregationBuilders.avg("avg_price").field("price")); + request.source(source); + + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 75L); + + assertNotNull(response); + assertEquals(75L, response.getTook().millis()); + assertNull(response.getAggregations()); + } + + public void testShardCountsWithNoAggregations() throws Exception { + SearchRequest request = new SearchRequest(); + request.source(new SearchSourceBuilder()); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 10L); + + assertEquals(1, response.getTotalShards()); + assertEquals(1, response.getSuccessfulShards()); + assertEquals(0, response.getSkippedShards()); + assertEquals(0, response.getFailedShards()); + } + + public void testTimingPreserved() throws Exception { + SearchRequest request = new SearchRequest(); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response1 = SearchResponseBuilder.build(List.of(), request, registry, 0L); + assertEquals(0L, response1.getTook().millis()); + + SearchResponse response2 = SearchResponseBuilder.build(List.of(), request, registry, 999L); + assertEquals(999L, response2.getTook().millis()); + } + + public void testEmptyHitsAlwaysReturned() throws Exception { + SearchRequest request = new SearchRequest(); + AggregationRegistry registry = new AggregationRegistry(); + + SearchResponse response = SearchResponseBuilder.build(List.of(), request, registry, 10L); + + assertNotNull(response.getHits()); + assertEquals(0, response.getHits().getHits().length); + assertNotNull(response.getHits().getTotalHits()); + } + + /** + * Regression: granularity keys must match even when the schema's column order opposes + * the request's nesting order (schema declares category before brand; request nests + * brand → category). Before key canonicalization this returned empty aggregations. + */ + public void testNestedGroupFieldsWithOpposingSchemaOrder() throws Exception { + Map mapping = new java.util.LinkedHashMap<>(); + mapping.put("category", "VARCHAR"); // lower column index than brand — the crux + mapping.put("brand", "VARCHAR"); + mapping.put("price", "INTEGER"); + CalciteTestInfra.InfraResult infra = CalciteTestInfra.buildFromMapping("products", mapping); + + SearchSourceBuilder source = new SearchSourceBuilder().size(0) + .aggregation( + AggregationBuilders.terms("by_brand") + .field("brand") + .subAggregation( + AggregationBuilders.terms("by_category") + .field("category") + .subAggregation(AggregationBuilders.avg("avg_price").field("price")) + ) + ); + + SearchSourceConverter converter = new SearchSourceConverter(infra.schema()); + QueryPlans plans = converter.convert(source, "products"); + List aggPlans = plans.get(QueryPlans.Type.AGGREGATION); + assertEquals(2, aggPlans.size()); + + List results = new ArrayList<>(); + for (QueryPlans.QueryPlan plan : aggPlans) { + List fields = plan.relNode().getRowType().getFieldNames(); + if (fields.contains("avg_price")) { + // Group columns arrive in schema order (category first) despite brand-first nesting. + assertEquals(List.of("category", "brand", "avg_price", "_count"), fields); + results.add( + new ExecutionResult( + plan, + List.of(new Object[] { "Cat1", "BrandA", 850.0, 2L }, new Object[] { "Cat2", "BrandA", 700.0, 1L }) + ) + ); + } else { + assertEquals(List.of("brand", "_count"), fields); + results.add(new ExecutionResult(plan, List.of(new Object[] { "BrandA", 3L }))); + } + } + + SearchRequest request = new SearchRequest("products"); + request.source(source); + SearchResponse response = SearchResponseBuilder.build(results, request, converter.getAggregationRegistry(), 1L); + + StringTerms byBrand = response.getAggregations().get("by_brand"); + assertNotNull("by_brand must be present", byBrand); + assertEquals(1, byBrand.getBuckets().size()); + assertEquals("BrandA", byBrand.getBuckets().get(0).getKeyAsString()); + assertEquals(3L, byBrand.getBuckets().get(0).getDocCount()); + + StringTerms byCategory = byBrand.getBuckets().get(0).getAggregations().get("by_category"); + assertNotNull("by_category must be present inside the brand bucket", byCategory); + assertEquals(2, byCategory.getBuckets().size()); + assertEquals("Cat1", byCategory.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, byCategory.getBuckets().get(0).getDocCount()); + InternalAvg avg1 = byCategory.getBuckets().get(0).getAggregations().get("avg_price"); + assertEquals(850.0, avg1.getValue(), 0.0); + InternalAvg avg2 = byCategory.getBuckets().get(1).getAggregations().get("avg_price"); + assertEquals(700.0, avg2.getValue(), 0.0); } // ---- Golden file driven SearchResponse generation tests ---- @@ -91,7 +244,14 @@ public void testGoldenFileSearchResponseGeneration() throws Exception { ExecutionResult result = new ExecutionResult(matchingPlans.get(0), rows); // Build and serialize SearchResponse - SearchResponse response = SearchResponseBuilder.build(List.of(result), 0L); + SearchRequest searchRequest = new SearchRequest(tc.getIndexName()); + searchRequest.source(searchSource); + SearchResponse response = SearchResponseBuilder.build( + List.of(result), + searchRequest, + converter.getAggregationRegistry(), + 0L + ); String responseJson = Strings.toString(MediaTypeRegistry.JSON, response); Map actualOutput = XContentHelper.convertToMap(JsonXContent.jsonXContent, responseJson, false); diff --git a/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/util/ComparisonUtilsTests.java b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/util/ComparisonUtilsTests.java new file mode 100644 index 0000000000000..be430ccabbb89 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/java/org/opensearch/dsl/util/ComparisonUtilsTests.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dsl.util; + +import org.opensearch.test.OpenSearchTestCase; + +public class ComparisonUtilsTests extends OpenSearchTestCase { + + public void testBothNull() { + assertTrue(ComparisonUtils.valuesEqual(null, null)); + } + + public void testFirstNull() { + assertFalse(ComparisonUtils.valuesEqual(null, "value")); + } + + public void testSecondNull() { + assertFalse(ComparisonUtils.valuesEqual("value", null)); + } + + public void testEqualStrings() { + assertTrue(ComparisonUtils.valuesEqual("test", "test")); + } + + public void testDifferentValues() { + assertFalse(ComparisonUtils.valuesEqual("foo", "bar")); + } + + public void testIntegerAndLong() { + assertTrue(ComparisonUtils.valuesEqual(42, 42L)); + } + + public void testIntegerAndDouble() { + assertTrue(ComparisonUtils.valuesEqual(42, 42.0)); + } + + public void testDoubleAndFloat() { + assertTrue(ComparisonUtils.valuesEqual(42.0, 42.0f)); + } + + public void testDifferentNumbers() { + assertFalse(ComparisonUtils.valuesEqual(42, 43)); + } + + public void testNumberAndStringAreNotEqual() { + // No cross-type stringification: bucket keys are schema-typed, "42" != 42. + assertFalse(ComparisonUtils.valuesEqual(42, "42")); + } + + public void testLargeLongsBeyondDoublePrecision() { + // 2^53 + 1 vs 2^53: indistinguishable as doubles, distinct as longs. + assertFalse(ComparisonUtils.valuesEqual(9007199254740993L, 9007199254740992L)); + assertTrue(ComparisonUtils.valuesEqual(9007199254740993L, 9007199254740993L)); + } + + public void testBooleanComparison() { + assertTrue(ComparisonUtils.valuesEqual(true, true)); + assertFalse(ComparisonUtils.valuesEqual(true, false)); + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/avg_with_missing_format_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/avg_with_missing_format_aggregation.json new file mode 100644 index 0000000000000..3727e551c8d9c --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/avg_with_missing_format_aggregation.json @@ -0,0 +1,49 @@ +{ + "testName": "avg_with_missing_format_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "avg_rating": { + "avg": { + "field": "rating", + "missing": 0, + "format": "0.00" + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], avg_rating=[AVG($4)])", + " LogicalProject(name=[$0], price=[$1], brand=[$2], rating=[$3], _missing0=[COALESCE($3, 0.0E0:DOUBLE)])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["avg_rating"], + "mockResultRows": [ + [3.625] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "avg_rating": { + "value": 3.625, + "value_as_string": "3.62" + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/extended_stats_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/extended_stats_aggregation.json new file mode 100644 index 0000000000000..f948ce29669f3 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/extended_stats_aggregation.json @@ -0,0 +1,65 @@ +{ + "testName": "extended_stats_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "price_stats": { + "extended_stats": { + "field": "price", + "sigma": 3.0 + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], price_stats_count=[COUNT($1)], price_stats_min=[MIN($1)], price_stats_max=[MAX($1)], price_stats_sum=[SUM($1)], price_stats_variance=[VAR_POP($1)])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["price_stats_count", "price_stats_min", "price_stats_max", "price_stats_sum", "price_stats_variance"], + "mockResultRows": [ + [4, 2, 8, 20, 4.0] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "price_stats": { + "count": 4, + "min": 2.0, + "max": 8.0, + "avg": 5.0, + "sum": 20.0, + "sum_of_squares": 116.0, + "variance": 4.0, + "variance_population": 4.0, + "variance_sampling": 5.333333333333333, + "std_deviation": 2.0, + "std_deviation_population": 2.0, + "std_deviation_sampling": 2.309401076758503, + "std_deviation_bounds": { + "upper": 11.0, + "lower": -1.0, + "upper_population": 11.0, + "lower_population": -1.0, + "upper_sampling": 11.928203230275509, + "lower_sampling": -1.9282032302755088 + } + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_aggregation.json new file mode 100644 index 0000000000000..0a4aa319f1540 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_aggregation.json @@ -0,0 +1,51 @@ +{ + "testName": "percentiles_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "pcts": { + "percentiles": { + "field": "price", + "percents": [50, 95, 99] + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], pcts_p50_0=[PERCENTILE_APPROX(APPROXIMATE $1, $4)], pcts_p95_0=[PERCENTILE_APPROX(APPROXIMATE $1, $5)], pcts_p99_0=[PERCENTILE_APPROX(APPROXIMATE $1, $6)])", + " LogicalProject(name=[$0], price=[$1], brand=[$2], rating=[$3], _lit0=[50.0E0:DOUBLE], _lit1=[95.0E0:DOUBLE], _lit2=[99.0E0:DOUBLE])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["pcts_p50_0", "pcts_p95_0", "pcts_p99_0"], + "mockResultRows": [ + [899.0, 1287.0, 1299.0] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "pcts": { + "values": { + "50.0": 899.0, + "95.0": 1287.0, + "99.0": 1299.0 + } + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_compression_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_compression_aggregation.json new file mode 100644 index 0000000000000..c4dd42fa0e664 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_compression_aggregation.json @@ -0,0 +1,53 @@ +{ + "testName": "percentiles_with_compression_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "pcts": { + "percentiles": { + "field": "price", + "percents": [50, 99], + "tdigest": { + "compression": 200 + } + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], pcts_p50_0=[PERCENTILE_APPROX_N(APPROXIMATE $1, $5, $4)], pcts_p99_0=[PERCENTILE_APPROX_N(APPROXIMATE $1, $6, $4)])", + " LogicalProject(name=[$0], price=[$1], brand=[$2], rating=[$3], _lit0=[200], _lit1=[50.0E0:DOUBLE], _lit2=[99.0E0:DOUBLE])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["pcts_p50_0", "pcts_p99_0"], + "mockResultRows": [ + [905.0, 1290.0] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "pcts": { + "values": { + "50.0": 905.0, + "99.0": 1290.0 + } + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_missing_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_missing_aggregation.json new file mode 100644 index 0000000000000..05384ab23c291 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/percentiles_with_missing_aggregation.json @@ -0,0 +1,54 @@ +{ + "testName": "percentiles_with_missing_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "pcts": { + "percentiles": { + "field": "rating", + "percents": [50, 95], + "missing": 0, + "format": "0.00" + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], pcts_p50_0=[PERCENTILE_APPROX(APPROXIMATE $4, $5)], pcts_p95_0=[PERCENTILE_APPROX(APPROXIMATE $4, $6)])", + " LogicalProject(name=[$0], price=[$1], brand=[$2], rating=[$3], _missing0=[COALESCE($3, 0.0E0:DOUBLE)], _lit1=[50.0E0:DOUBLE], _lit2=[95.0E0:DOUBLE])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["pcts_p50_0", "pcts_p95_0"], + "mockResultRows": [ + [750.5, 1200.25] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "pcts": { + "values": { + "50.0": 750.5, + "50.0_as_string": "750.50", + "95.0": 1200.25, + "95.0_as_string": "1200.25" + } + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/stats_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/stats_aggregation.json new file mode 100644 index 0000000000000..7fc545379277f --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/stats_aggregation.json @@ -0,0 +1,49 @@ +{ + "testName": "stats_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "price_stats": { + "stats": { + "field": "price" + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], price_stats_count=[COUNT($1)], price_stats_min=[MIN($1)], price_stats_max=[MAX($1)], price_stats_sum=[SUM($1)])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["price_stats_count", "price_stats_min", "price_stats_max", "price_stats_sum"], + "mockResultRows": [ + [3, 100, 600, 900] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "price_stats": { + "count": 3, + "min": 100.0, + "max": 600.0, + "avg": 300.0, + "sum": 900.0 + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_numeric_keys_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_numeric_keys_aggregation.json new file mode 100644 index 0000000000000..0d0dab2ffe92a --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_numeric_keys_aggregation.json @@ -0,0 +1,58 @@ +{ + "testName": "terms_numeric_keys_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "by_price": { + "terms": { + "field": "price" + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalSort(sort0=[$1], sort1=[$0], dir0=[DESC], dir1=[ASC])", + " LogicalAggregate(group=[{1}], _count=[COUNT()])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["price", "_count"], + "mockResultRows": [ + [100, 3], + [200, 2] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "by_price": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 100, + "doc_count": 3 + }, + { + "key": 200, + "doc_count": 2 + } + ] + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json index 1c9838bf2551c..cd22ad89c9a9a 100644 --- a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_avg_aggregation.json @@ -44,6 +44,28 @@ }, "max_score": 0.0, "hits": [] + }, + "aggregations": { + "by_brand": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "BrandA", + "doc_count": 3, + "avg_price": { + "value": 850.0 + } + }, + { + "key": "BrandB", + "doc_count": 2, + "avg_price": { + "value": 1100.0 + } + } + ] + } } } } diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_percentiles_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_percentiles_aggregation.json new file mode 100644 index 0000000000000..1c416285a63a2 --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_percentiles_aggregation.json @@ -0,0 +1,77 @@ +{ + "testName": "terms_with_percentiles_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "by_brand": { + "terms": { + "field": "brand" + }, + "aggregations": { + "pcts": { + "percentiles": { + "field": "price", + "percents": [50] + } + } + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalSort(sort0=[$2], sort1=[$0], dir0=[DESC], dir1=[ASC])", + " LogicalAggregate(group=[{2}], pcts_p50_0=[PERCENTILE_APPROX(APPROXIMATE $1, $4)], _count=[COUNT()])", + " LogicalProject(name=[$0], price=[$1], brand=[$2], rating=[$3], _lit0=[50.0E0:DOUBLE])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["brand", "pcts_p50_0", "_count"], + "mockResultRows": [ + ["BrandA", 700.0, 3], + ["BrandB", 1100.0, 2] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "by_brand": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "BrandA", + "doc_count": 3, + "pcts": { + "values": { + "50.0": 700.0 + } + } + }, + { + "key": "BrandB", + "doc_count": 2, + "pcts": { + "values": { + "50.0": 1100.0 + } + } + } + ] + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_stats_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_stats_aggregation.json new file mode 100644 index 0000000000000..fe274160280dd --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/terms_with_stats_aggregation.json @@ -0,0 +1,79 @@ +{ + "testName": "terms_with_stats_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "by_brand": { + "terms": { + "field": "brand" + }, + "aggregations": { + "price_stats": { + "stats": { + "field": "price" + } + } + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalSort(sort0=[$5], sort1=[$0], dir0=[DESC], dir1=[ASC])", + " LogicalAggregate(group=[{2}], price_stats_count=[COUNT($1)], price_stats_min=[MIN($1)], price_stats_max=[MAX($1)], price_stats_sum=[SUM($1)], _count=[COUNT()])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["brand", "price_stats_count", "price_stats_min", "price_stats_max", "price_stats_sum", "_count"], + "mockResultRows": [ + ["BrandA", 3, 100, 600, 900, 3], + ["BrandB", 2, 1000, 1200, 2200, 2] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "by_brand": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "BrandA", + "doc_count": 3, + "price_stats": { + "count": 3, + "min": 100.0, + "max": 600.0, + "avg": 300.0, + "sum": 900.0 + } + }, + { + "key": "BrandB", + "doc_count": 2, + "price_stats": { + "count": 2, + "min": 1000.0, + "max": 1200.0, + "avg": 1100.0, + "sum": 2200.0 + } + } + ] + } + } + } +} diff --git a/sandbox/plugins/dsl-query-executor/src/test/resources/golden/value_count_aggregation.json b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/value_count_aggregation.json new file mode 100644 index 0000000000000..ecc9aa0c8f8dc --- /dev/null +++ b/sandbox/plugins/dsl-query-executor/src/test/resources/golden/value_count_aggregation.json @@ -0,0 +1,45 @@ +{ + "testName": "value_count_aggregation", + "indexName": "test-index", + "indexMapping": { + "name": "VARCHAR", + "price": "INTEGER", + "brand": "VARCHAR", + "rating": "DOUBLE" + }, + "planType": "AGGREGATION", + "inputDsl": { + "size": 0, + "aggregations": { + "price_count": { + "value_count": { + "field": "price" + } + } + } + }, + "expectedRelNodePlan": [ + "LogicalAggregate(group=[{}], price_count=[COUNT($1)])", + " LogicalTableScan(table=[[test-index]])" + ], + "mockResultFieldNames": ["price_count"], + "mockResultRows": [ + [5] + ], + "expectedOutputDsl": { + "num_reduce_phases": 0, + "hits": { + "total": { + "value": 0, + "relation": "eq" + }, + "max_score": 0.0, + "hits": [] + }, + "aggregations": { + "price_count": { + "value": 5 + } + } + } +}