diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java index 6c543ddc8c3..c7cebd4bba0 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java @@ -56,6 +56,7 @@ public enum CommandType { public enum Option { countField, showCount, + showPerc, useNull, } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 94c40e5adb2..8bed47640d2 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -3230,6 +3230,39 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { orderKeys.add(countField); orderKeys.addAll(tieBreakKeys); + // 3. compute percentage if showperc=true + Boolean showPerc = (Boolean) argumentMap.get(RareTopN.Option.showPerc.name()).getValue(); + if (showPerc) { + if (context.relBuilder.peek().getRowType().getFieldNames().contains("percent")) { + throw new IllegalArgumentException( + "Field `percent` already exists in the output. Consider renaming the conflicting" + + " field."); + } + RexNode totalWindowOver = + PlanUtils.makeOver( + context, + BuiltinFunctionName.SUM, + context.relBuilder.field(countFieldName), + List.of(), + partitionKeys, + List.of(), + WindowFrame.rowsUnbounded()); + + RexNode hundred = context.relBuilder.literal(new BigDecimal("100.0")); + RexNode countCast = + context.relBuilder.cast(context.relBuilder.field(countFieldName), SqlTypeName.DOUBLE); + RexNode totalCast = context.relBuilder.cast(totalWindowOver, SqlTypeName.DOUBLE); + RexNode numerator = context.relBuilder.call(SqlStdOperatorTable.MULTIPLY, hundred, countCast); + RexNode percValue = context.relBuilder.call(SqlStdOperatorTable.DIVIDE, numerator, totalCast); + + // Round the percent value 2 decimal places + RexNode roundedPerc = + context.relBuilder.call( + SqlStdOperatorTable.ROUND, percValue, context.relBuilder.literal(2)); + + context.relBuilder.projectPlus(context.relBuilder.alias(roundedPerc, "percent")); + } + RexNode rowNumberWindowOver = PlanUtils.makeOver( context, @@ -3242,14 +3275,14 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) { context.relBuilder.projectPlus( context.relBuilder.alias(rowNumberWindowOver, ROW_NUMBER_COLUMN_FOR_RARE_TOP)); - // 3. filter row_number() <= k in each partition + // 4. filter row_number() <= k in each partition int k = node.getNoOfResults(); context.relBuilder.filter( context.relBuilder.lessThanOrEqual( context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP), context.relBuilder.literal(k))); - // 4. project final output. the default output is group by list + field list + // 5. project final output: group by list + field list, optionally count and percent Boolean showCount = (Boolean) argumentMap.get(RareTopN.Option.showCount.name()).getValue(); if (showCount) { context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP)); diff --git a/docs/user/ppl/cmd/rare.md b/docs/user/ppl/cmd/rare.md index 993e1a9c987..89bb8fea29b 100644 --- a/docs/user/ppl/cmd/rare.md +++ b/docs/user/ppl/cmd/rare.md @@ -23,7 +23,7 @@ The `rare` command supports the following parameters. | --- | --- | --- | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | -| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `showperc`: Whether to create a field in the output containing the percentage of each value's count relative to the total. Default is `false`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | ## Example 1: Finding the least common values without showing counts @@ -125,7 +125,30 @@ fetched rows / total rows = 4/4 +--------------+-----+ ``` -## Example 5: Specifying null value handling +## Example 5: Displaying percentages + + The following query finds the least common severity levels and shows what percentage each represents: + +```ppl +source=otellogs +| rare showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+---------+ +``` + +## Example 6: Specifying null value handling The following query uses `usenull=false` to exclude null values: @@ -166,4 +189,4 @@ fetched rows / total rows = 4/4 | @opentelemetry/instrumentation-http | 2 | | null | 16 | +-----------------------------------------------------------------------------+-------+ -``` \ No newline at end of file +``` diff --git a/docs/user/ppl/cmd/top.md b/docs/user/ppl/cmd/top.md index 678b62354a0..ae4b4735979 100644 --- a/docs/user/ppl/cmd/top.md +++ b/docs/user/ppl/cmd/top.md @@ -20,7 +20,7 @@ The `top` command supports the following parameters. | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Optional | The number of results to return. Default is `10`. | -| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`showperc`: Whether to create a field in the output that represents the percentage of the count relative to the total. Default is `false`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | @@ -139,7 +139,30 @@ fetched rows / total rows = 7/7 +----------------------------------+--------------+ ``` -## Example 6: Specifying null value handling +## Example 6: Displaying percentages + +The following query finds the most common severity levels and shows the percentage each represents: + +```ppl +source=otellogs +| top showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+---------+ +``` + +## Example 7: Specifying null value handling The following query specifies `usenull=false` to exclude null values: diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java index e555576a9cd..dcbb7196546 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java @@ -6,9 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; import org.json.JSONObject; @@ -41,6 +39,43 @@ public void testTopCommandUseNullFalse() throws IOException { verifyNumOfRows(result, 5); } + @Test + public void testTopCommandShowPerc() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | top showperc=true age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder( + result, schema("age", "int"), schema("count", "bigint"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 2, 28.57), + rows(28, 1, 14.29), + rows(32, 1, 14.29), + rows(33, 1, 14.29), + rows(34, 1, 14.29), + rows(null, 1, 14.29)); + } + + @Test + public void testTopCommandShowPercWithoutShowCount() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | top showperc=true showcount=false age", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 28.57), + rows(28, 14.29), + rows(32, 14.29), + rows(33, 14.29), + rows(34, 14.29), + rows(null, 14.29)); + } + @Test public void testTopCommandLegacyFalse() throws IOException { withSettings( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java index 936e728bd10..8121ce0ea6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java @@ -58,4 +58,20 @@ public void testTopNWithGroup() throws IOException { verifyDataRows(result, rows("F", "TX"), rows("M", "MD")); } } + + @Test + public void testTopWithShowPerc() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | top showperc=true gender", TEST_INDEX_ACCOUNT)); + if (isCalciteEnabled()) { + verifySchemaInOrder( + result, + schema("gender", "string"), + schema("count", "bigint"), + schema("percent", "double")); + verifyDataRows(result, rows("M", 507, 50.7), rows("F", 493, 49.3)); + } else { + verifyDataRows(result, rows("M"), rows("F")); + } + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index b26751ad61b..34b8f0287da 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -186,6 +186,7 @@ UNION: 'UNION'; MAXOUT: 'MAXOUT'; COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; +SHOWPERC: 'SHOWPERC'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; OTHERSTR: 'OTHERSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index eeaed6daf52..57bc618cc36 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -487,6 +487,7 @@ rareTopCommand rareTopOption : COUNTFIELD EQUAL countField = stringLiteral | SHOWCOUNT EQUAL showCount = booleanLiteral + | SHOWPERC EQUAL showPerc = booleanLiteral | USENULL EQUAL useNull = booleanLiteral ; @@ -1791,6 +1792,7 @@ searchableKeyWord | ANOMALY_SCORE_THRESHOLD | COUNTFIELD | SHOWCOUNT + | SHOWPERC | MAXOUT | PATH | INPUT diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 2cdc702b785..54bb2d4fa3e 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -307,6 +307,11 @@ public static List getArgumentList( new Argument( RareTopN.Option.showCount.name(), opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE)); + opt = ctx.rareTopOption().stream().filter(op -> op.showPerc != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.showPerc.name(), + opt.isPresent() ? getArgumentValue(opt.get().showPerc) : Literal.FALSE)); opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst(); list.add( new Argument( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 11c47d137e8..c0ee4d3ef6a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -459,13 +459,15 @@ public String visitRareTopN(RareTopN node, String context) { Integer noOfResults = node.getNoOfResults(); String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue(); Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue(); + Boolean showPerc = (Boolean) arguments.get(RareTopN.Option.showPerc.name()).getValue(); Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue(); String fields = visitFieldList(node.getFields()); String group = visitExpressionList(node.getGroupExprList()); String options = UnresolvedPlanHelper.isCalciteEnabled(settings) ? StringUtils.format( - "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull) + "countfield='%s' showcount=%s showperc=%s usenull=%s ", + countField, showCount, showPerc, useNull) : ""; return StringUtils.format( "%s | %s %d %s%s", diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 0c5ad9dd4fe..ac1a2141542 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -398,6 +398,18 @@ public void testRareCommandWithGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testRareCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | rare showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | rare showperc=true a by b"); + assertNotEquals(null, tree); + } + @Test public void testTopCommandWithoutNShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | top a"); @@ -422,6 +434,18 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testTopCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a by b"); + assertNotEquals(null, tree); + } + @Test public void testCanParseMultiMatchRelevanceFunction() { assertNotEquals( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 21aa15c2e64..48256d28549 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -222,6 +222,166 @@ public void failWithDuplicatedName() { } } + @Test + public void testRareShowPerc() { + String ppl = "source=EMP | rare showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; percent=7.14\n" + + "JOB=ANALYST; count=2; percent=14.29\n" + + "JOB=MANAGER; count=3; percent=21.43\n" + + "JOB=CLERK; count=4; percent=28.57\n" + + "JOB=SALESMAN; count=4; percent=28.57\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithGroupBy() { + String ppl = "source=EMP | rare showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.33\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.33\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.33\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.67\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.67\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.67\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` NULLS LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithoutShowCount() { + String ppl = "source=EMP | rare showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; percent=7.14\n" + + "JOB=ANALYST; percent=14.29\n" + + "JOB=MANAGER; percent=21.43\n" + + "JOB=CLERK; percent=28.57\n" + + "JOB=SALESMAN; percent=28.57\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithLimit() { + String ppl = "source=EMP | rare 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // PRESIDENT has 1 out of 14 total employees = 7.14% + String expectedResult = "JOB=PRESIDENT; count=1; percent=7.14\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTop() { String ppl = "source=EMP | top JOB"; @@ -408,4 +568,165 @@ public void testTopUseNullFalse() { + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testTopShowPerc() { + String ppl = "source=EMP | top showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; percent=28.57\n" + + "JOB=SALESMAN; count=4; percent=28.57\n" + + "JOB=MANAGER; count=3; percent=21.43\n" + + "JOB=ANALYST; count=2; percent=14.29\n" + + "JOB=PRESIDENT; count=1; percent=7.14\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithGroupBy() { + String ppl = "source=EMP | top showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.33\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.33\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.33\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.67\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.67\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.67\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` DESC NULLS FIRST, `JOB` NULLS LAST)" + + " `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithoutShowCount() { + String ppl = "source=EMP | top showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; percent=28.57\n" + + "JOB=SALESMAN; percent=28.57\n" + + "JOB=MANAGER; percent=21.43\n" + + "JOB=ANALYST; percent=14.29\n" + + "JOB=PRESIDENT; percent=7.14\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithLimit() { + String ppl = "source=EMP | top 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 2)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // CLERK has 4 out of 14 total employees = 28.57% + String expectedResult = "JOB=CLERK; count=4; percent=28.57\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 2) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5f45a96f8c..bd1bb0ec7ed 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -740,6 +740,7 @@ public void testRareCommand() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -756,6 +757,7 @@ public void testRareCommandWithGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -772,12 +774,47 @@ public void testRareCommandWithMultipleFields() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); } + @Test + public void testRareCommandWithShowPerc() { + assertEqual( + "source=t | rare showperc=true a", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testRareCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | rare showperc=true a by b", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithN() { assertEqual( @@ -789,6 +826,7 @@ public void testTopCommandWithN() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -805,6 +843,7 @@ public void testTopCommandWithoutNAndGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -821,6 +860,7 @@ public void testTopCommandWithNAndGroupBy() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -837,6 +877,7 @@ public void testTopCommandWithMultipleFields() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), @@ -854,11 +895,46 @@ public void testTopCommandWithUseNullFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); } + @Test + public void testTopCommandWithShowPerc() { + assertEqual( + "source=t | top showperc=true a", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testTopCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | top showperc=true a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithLegacyFalse() { when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); @@ -871,6 +947,7 @@ public void testTopCommandWithLegacyFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index eba4f57112b..9a0d4c58739 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -406,20 +406,38 @@ public void testTopCommandWithNAndGroupBy() { public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | rare 10 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | rare 10 countfield='count' showcount=true showperc=false usenull=true" + + " identifier by identifier", anonymize("source=t | rare a by b")); } @Test - public void testTopCommandWithNAndGroupByWithCalcite() { + public void testRareCommandWithShowPercWithCalCite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | top 1 countield='count' showcount=true usenull=true identifier by" + "source=table | rare 10 countfield='count' showcount=true showperc=true usenull=true" + " identifier", + anonymize("source=t | rare showperc=true a ")); + } + + @Test + public void testTopCommandWithNAndGroupByWithCalcite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | top 1 countfield='count' showcount=true showperc=false usenull=true" + + " identifier by identifier", anonymize("source=t | top 1 a by b")); } + @Test + public void testTopCommandWithShowPercWithCalcite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | top 1 countfield='count' showcount=true showperc=true usenull=true" + + " identifier by identifier", + anonymize("source=t | top 1 showperc=true a by b")); + } + @Test public void testAndExpression() { assertEquals(