From 84c60949ab01e36fcc87f0267a60975635c88f6b Mon Sep 17 00:00:00 2001 From: Ajimelec Gonzalez Date: Tue, 21 Jul 2026 14:06:47 -0700 Subject: [PATCH] [Feature] `showperc` with `top` and `rare` commands. * Clean docs * Fix CalcitePPLRareTopNTest showperc tests * Calculate percentages before filtering * Added percentfield and changed decimal places from 2 to 6 Signed-off-by: Ajimelec Gonzalez --- .../org/opensearch/sql/ast/tree/RareTopN.java | 2 + .../sql/calcite/CalciteRelNodeVisitor.java | 45 +- docs/user/ppl/cmd/rare.md | 51 ++- docs/user/ppl/cmd/top.md | 49 ++- .../calcite/remote/CalciteTopCommandIT.java | 41 +- .../org/opensearch/sql/ppl/TopCommandIT.java | 16 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 + .../sql/ppl/utils/ArgumentFactory.java | 12 + .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 5 +- .../sql/ppl/antlr/PPLSyntaxParserTest.java | 38 ++ .../ppl/calcite/CalcitePPLRareTopNTest.java | 401 ++++++++++++++++++ .../sql/ppl/parser/AstBuilderTest.java | 90 ++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 26 +- 14 files changed, 767 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java index 6c543ddc8c3..8055c4f92d8 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java @@ -56,6 +56,8 @@ public enum CommandType { public enum Option { countField, showCount, + percentField, + showPerc, useNull, } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 94c40e5adb2..e7e4058d644 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -221,6 +221,8 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | -| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `percentfield`: The name of the field that contains the percentage. Default is `percent`.
- `showperc`: Whether to create a field in the output containing the percentage of each value's count relative to the total. Default is `false`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | ## Example 1: Finding the least common values without showing counts @@ -125,7 +125,52 @@ fetched rows / total rows = 4/4 +--------------+-----+ ``` -## Example 5: Specifying null value handling +## Example 5: Displaying percentages + + The following query finds the least common severity levels and shows what percentage each represents: + +```ppl +source=otellogs +| rare showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+---------+ +``` + +## Example 6: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| rare showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+------+ +``` + +## Example 7: Specifying null value handling The following query uses `usenull=false` to exclude null values: @@ -166,4 +211,4 @@ fetched rows / total rows = 4/4 | @opentelemetry/instrumentation-http | 2 | | null | 16 | +-----------------------------------------------------------------------------+-------+ -``` \ No newline at end of file +``` diff --git a/docs/user/ppl/cmd/top.md b/docs/user/ppl/cmd/top.md index 678b62354a0..162ad56e905 100644 --- a/docs/user/ppl/cmd/top.md +++ b/docs/user/ppl/cmd/top.md @@ -20,7 +20,7 @@ The `top` command supports the following parameters. | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Optional | The number of results to return. Default is `10`. | -| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`percentfield`: The name of the field that contains the percentage. Default is `percent`.
`showperc`: Whether to create a field in the output that represents the percentage of the count relative to the total. Default is `false`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | @@ -139,7 +139,52 @@ fetched rows / total rows = 7/7 +----------------------------------+--------------+ ``` -## Example 6: Specifying null value handling +## Example 6: Displaying percentages + +The following query finds the most common severity levels and shows the percentage each represents: + +```ppl +source=otellogs +| top showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+---------+ +``` + +## Example 7: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| top showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+------+ +``` + +## Example 8: Specifying null value handling The following query specifies `usenull=false` to exclude null values: diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java index e555576a9cd..bb39d3c5460 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java @@ -6,9 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; import org.json.JSONObject; @@ -41,6 +39,43 @@ public void testTopCommandUseNullFalse() throws IOException { verifyNumOfRows(result, 5); } + @Test + public void testTopCommandShowPerc() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | top showperc=true age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder( + result, schema("age", "int"), schema("count", "bigint"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 2, 28.571429), + rows(28, 1, 14.285714), + rows(32, 1, 14.285714), + rows(33, 1, 14.285714), + rows(34, 1, 14.285714), + rows(null, 1, 14.285714)); + } + + @Test + public void testTopCommandShowPercWithoutShowCount() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | top showperc=true showcount=false age", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 28.571429), + rows(28, 14.285714), + rows(32, 14.285714), + rows(33, 14.285714), + rows(34, 14.285714), + rows(null, 14.285714)); + } + @Test public void testTopCommandLegacyFalse() throws IOException { withSettings( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java index 936e728bd10..8121ce0ea6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java @@ -58,4 +58,20 @@ public void testTopNWithGroup() throws IOException { verifyDataRows(result, rows("F", "TX"), rows("M", "MD")); } } + + @Test + public void testTopWithShowPerc() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | top showperc=true gender", TEST_INDEX_ACCOUNT)); + if (isCalciteEnabled()) { + verifySchemaInOrder( + result, + schema("gender", "string"), + schema("count", "bigint"), + schema("percent", "double")); + verifyDataRows(result, rows("M", 507, 50.7), rows("F", 493, 49.3)); + } else { + verifyDataRows(result, rows("M"), rows("F")); + } + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index b26751ad61b..4cc72ac6540 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -186,6 +186,8 @@ UNION: 'UNION'; MAXOUT: 'MAXOUT'; COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; +PERCENTFIELD: 'PERCENTFIELD'; +SHOWPERC: 'SHOWPERC'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; OTHERSTR: 'OTHERSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index eeaed6daf52..e5f111e27a4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -487,6 +487,8 @@ rareTopCommand rareTopOption : COUNTFIELD EQUAL countField = stringLiteral | SHOWCOUNT EQUAL showCount = booleanLiteral + | PERCENTFIELD EQUAL percentField = stringLiteral + | SHOWPERC EQUAL showPerc = booleanLiteral | USENULL EQUAL useNull = booleanLiteral ; @@ -1791,6 +1793,8 @@ searchableKeyWord | ANOMALY_SCORE_THRESHOLD | COUNTFIELD | SHOWCOUNT + | PERCENTFIELD + | SHOWPERC | MAXOUT | PATH | INPUT diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 2cdc702b785..3aba4faf99c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -307,6 +307,18 @@ public static List getArgumentList( new Argument( RareTopN.Option.showCount.name(), opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE)); + opt = ctx.rareTopOption().stream().filter(op -> op.percentField != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.percentField.name(), + opt.isPresent() + ? getArgumentValue(opt.get().percentField) + : new Literal("percent", DataType.STRING))); + opt = ctx.rareTopOption().stream().filter(op -> op.showPerc != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.showPerc.name(), + opt.isPresent() ? getArgumentValue(opt.get().showPerc) : Literal.FALSE)); opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst(); list.add( new Argument( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 11c47d137e8..5c5ebd20130 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -459,13 +459,16 @@ public String visitRareTopN(RareTopN node, String context) { Integer noOfResults = node.getNoOfResults(); String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue(); Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue(); + String percentField = (String) arguments.get(RareTopN.Option.percentField.name()).getValue(); + Boolean showPerc = (Boolean) arguments.get(RareTopN.Option.showPerc.name()).getValue(); Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue(); String fields = visitFieldList(node.getFields()); String group = visitExpressionList(node.getGroupExprList()); String options = UnresolvedPlanHelper.isCalciteEnabled(settings) ? StringUtils.format( - "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull) + "countfield='%s' showcount=%s percentfield='%s' showperc=%s usenull=%s ", + countField, showCount, percentField, showPerc, useNull) : ""; return StringUtils.format( "%s | %s %d %s%s", diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 0c5ad9dd4fe..42abefc8dc6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -398,6 +398,25 @@ public void testRareCommandWithGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testRareCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | rare showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | rare showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | rare showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testTopCommandWithoutNShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | top a"); @@ -422,6 +441,25 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testTopCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | top showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testCanParseMultiMatchRelevanceFunction() { assertNotEquals( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 21aa15c2e64..f8faa2a70d9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -222,6 +222,206 @@ public void failWithDuplicatedName() { } } + @Test + public void testRareShowPerc() { + String ppl = "source=EMP | rare showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; percent=7.142857\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithGroupBy() { + String ppl = "source=EMP | rare showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` NULLS LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithoutShowCount() { + String ppl = "source=EMP | rare showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; percent=7.142857\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercCustomField() { + String ppl = "source=EMP | rare percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; pct=7.142857\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithLimit() { + String ppl = "source=EMP | rare 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // PRESIDENT has 1 out of 14 total employees = 7.142857% + String expectedResult = "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTop() { String ppl = "source=EMP | top JOB"; @@ -408,4 +608,205 @@ public void testTopUseNullFalse() { + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testTopShowPerc() { + String ppl = "source=EMP | top showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithGroupBy() { + String ppl = "source=EMP | top showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` DESC NULLS FIRST, `JOB` NULLS LAST)" + + " `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithoutShowCount() { + String ppl = "source=EMP | top showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=PRESIDENT; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercCustomField() { + String ppl = "source=EMP | top percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=PRESIDENT; count=1; pct=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithLimit() { + String ppl = "source=EMP | top 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // CLERK has 4 out of 14 total employees = 28.571429% + String expectedResult = "JOB=CLERK; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5f45a96f8c..57e7dbb42a4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -740,6 +740,8 @@ public void testRareCommand() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -756,6 +758,8 @@ public void testRareCommandWithGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -772,12 +776,50 @@ public void testRareCommandWithMultipleFields() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); } + @Test + public void testRareCommandWithShowPerc() { + assertEqual( + "source=t | rare showperc=true a", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testRareCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | rare showperc=true a by b", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithN() { assertEqual( @@ -789,6 +831,8 @@ public void testTopCommandWithN() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -805,6 +849,8 @@ public void testTopCommandWithoutNAndGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -821,6 +867,8 @@ public void testTopCommandWithNAndGroupBy() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -837,6 +885,8 @@ public void testTopCommandWithMultipleFields() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), @@ -854,11 +904,49 @@ public void testTopCommandWithUseNullFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); } + @Test + public void testTopCommandWithShowPerc() { + assertEqual( + "source=t | top showperc=true a", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testTopCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | top showperc=true a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithLegacyFalse() { when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); @@ -871,6 +959,8 @@ public void testTopCommandWithLegacyFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index eba4f57112b..2025634e4a2 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -406,20 +406,38 @@ public void testTopCommandWithNAndGroupBy() { public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | rare 10 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | rare a by b")); } + @Test + public void testRareCommandWithShowPercWithCalCite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier", + anonymize("source=t | rare showperc=true a ")); + } + @Test public void testTopCommandWithNAndGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | top 1 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | top 1 a by b")); } + @Test + public void testTopCommandWithShowPercWithCalcite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier by identifier", + anonymize("source=t | top 1 showperc=true a by b")); + } + @Test public void testAndExpression() { assertEquals(