[Feature] percentfield and showperc with top and rare commands.

core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java

@@ -56,6 +56,8 @@ public enum CommandType {
   public enum Option {
     countField,
     showCount,
+    percentField,
+    showPerc,
     useNull,
   }
 }

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -223,6 +223,8 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalciteP
   private final MapPathPreMaterializer mapPathMaterializer;
   private final ForeachPlanner foreachPlanner;
 
+  private static final int PERCENT_DECIMAL_PLACES = 6;
+
   public CalciteRelNodeVisitor(DataSourceService dataSourceService) {
     this.rexVisitor = new CalciteRexNodeVisitor(this);
     this.aggVisitor = new CalciteAggCallVisitor(rexVisitor);
@@ -3264,6 +3266,45 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) {
     orderKeys.add(countField);
     orderKeys.addAll(tieBreakKeys);
 
+    // 3. compute percentage if showperc=true
+    Boolean showPerc = (Boolean) argumentMap.get(RareTopN.Option.showPerc.name()).getValue();
+    if (showPerc) {
+      String percentFieldName =
+          (String) argumentMap.get(RareTopN.Option.percentField.name()).getValue();
+
+      if (context.relBuilder.peek().getRowType().getFieldNames().contains(percentFieldName)) {
+        throw new IllegalArgumentException(
+            "Field `"
+                + percentFieldName
+                + "` already exists in the output. Consider renaming with percentfield='xyz'");
+      }
+      RexNode totalWindowOver =
+          PlanUtils.makeOver(
+              context,
+              BuiltinFunctionName.SUM,
+              context.relBuilder.field(countFieldName),
+              List.of(),
+              partitionKeys,
+              List.of(),
+              WindowFrame.rowsUnbounded());
+
+      RexNode hundred = context.relBuilder.literal(new BigDecimal("100.0"));
+      RexNode countCast =
+          context.relBuilder.cast(context.relBuilder.field(countFieldName), SqlTypeName.DOUBLE);
+      RexNode totalCast = context.relBuilder.cast(totalWindowOver, SqlTypeName.DOUBLE);
+      RexNode numerator = context.relBuilder.call(SqlStdOperatorTable.MULTIPLY, hundred, countCast);
+      RexNode percValue = context.relBuilder.call(SqlStdOperatorTable.DIVIDE, numerator, totalCast);
+
+      // Round the percent value 6 decimal places
+      RexNode roundedPerc =
+          context.relBuilder.call(
+              SqlStdOperatorTable.ROUND,
+              percValue,
+              context.relBuilder.literal(PERCENT_DECIMAL_PLACES));
+
+      context.relBuilder.projectPlus(context.relBuilder.alias(roundedPerc, percentFieldName));
+    }
+
     RexNode rowNumberWindowOver =
         PlanUtils.makeOver(
             context,
@@ -3276,14 +3317,14 @@ public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) {
     context.relBuilder.projectPlus(
         context.relBuilder.alias(rowNumberWindowOver, ROW_NUMBER_COLUMN_FOR_RARE_TOP));
 
-    // 3. filter row_number() <= k in each partition
+    // 4. filter row_number() <= k in each partition
     int k = node.getNoOfResults();
     context.relBuilder.filter(
         context.relBuilder.lessThanOrEqual(
             context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP),
             context.relBuilder.literal(k)));
 
-    // 4. project final output. the default output is group by list + field list
+    // 5. project final output: group by list + field list, optionally count and percent
     Boolean showCount = (Boolean) argumentMap.get(RareTopN.Option.showCount.name()).getValue();
     if (showCount) {
       context.relBuilder.projectExcept(context.relBuilder.field(ROW_NUMBER_COLUMN_FOR_RARE_TOP));

docs/user/ppl/cmd/rare.md

@@ -23,7 +23,7 @@ The `rare` command supports the following parameters.
 | --- | --- | --- |
 | `<field-list>` | Required | A comma-delimited list of field names. |
 | `<by-clause>` | Optional | One or more fields to group the results by. |
-| `rare-options` | Optional | Additional options for controlling output: <br> - `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`. <br> - `countfield`: The name of the field that contains the count. Default is `count`. <br> - `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. |
+| `rare-options` | Optional | Additional options for controlling output: <br> - `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`. <br> - `countfield`: The name of the field that contains the count. Default is `count`. <br> - `percentfield`: The name of the field that contains the percentage. Default is `percent`. <br> - `showperc`: Whether to create a field in the output containing the percentage of each value's count relative to the total. Default is `false`. <br> - `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. |
 
 ## Example 1: Finding the least common values without showing counts
 
@@ -125,7 +125,52 @@ fetched rows / total rows = 4/4
 +--------------+-----+
 ```
 
-## Example 5: Specifying null value handling
+## Example 5: Displaying percentages
+
+   The following query finds the least common severity levels and shows what percentage each represents:
+
+```ppl
+source=otellogs
+| rare showperc=true severityText
+```
+
+The query returns the following results:
+
+```text
+fetched rows / total rows = 4/4
++--------------+-------+---------+
+| severityText | count | percent |
+|--------------+-------+---------|
+| DEBUG        | 3     | 15.0    |
+| WARN         | 4     | 20.0    |
+| INFO         | 6     | 30.0    |
+| ERROR        | 7     | 35.0    |
++--------------+-------+---------+
+```
+
+## Example 6: Customizing the percent field name
+The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`:
+
+```ppl
+source=otellogs
+| rare showperc=true percentfield='pct' severityText
+```
+
+The query returns the following results:
+
+```text
+fetched rows / total rows = 4/4
++--------------+-------+------+
+| severityText | count | pct  |
+|--------------+-------+------|
+| DEBUG        | 3     | 15.0 |
+| WARN         | 4     | 20.0 |
+| INFO         | 6     | 30.0 |
+| ERROR        | 7     | 35.0 |
++--------------+-------+------+
+```
+
+## Example 7: Specifying null value handling
 
 The following query uses `usenull=false` to exclude null values:
 
@@ -166,4 +211,4 @@ fetched rows / total rows = 4/4
 | @opentelemetry/instrumentation-http                                         | 2     |
 | null                                                                        | 16    |
 +-----------------------------------------------------------------------------+-------+
-```
+```

docs/user/ppl/cmd/top.md

@@ -20,7 +20,7 @@ The `top` command supports the following parameters.
 | Parameter | Required/Optional | Description |
 | --- | --- | --- |
 | `<N>` | Optional | The number of results to return. Default is `10`. |
-| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.<br>`countfield`: The name of the field that contains the count. Default is `count`.<br>`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. |
+| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.<br>`countfield`: The name of the field that contains the count. Default is `count`.<br>`percentfield`: The name of the field that contains the percentage. Default is `percent`.<br>`showperc`: Whether to create a field in the output that represents the percentage of the count relative to the total. Default is `false`.<br>`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. |
 | `<field-list>` | Required | A comma-delimited list of field names.  |
 | `<by-clause>` | Optional | One or more fields to group the results by. |
 
@@ -139,7 +139,52 @@ fetched rows / total rows = 7/7
 +----------------------------------+--------------+
 ```
 
-## Example 6: Specifying null value handling
+## Example 6: Displaying percentages
+
+The following query finds the most common severity levels and shows the percentage each represents:
+
+```ppl
+source=otellogs
+| top showperc=true severityText
+```
+
+The query returns the following results:
+
+```text
+fetched rows / total rows = 4/4
++--------------+-------+---------+
+| severityText | count | percent |
+|--------------+-------+---------|
+| ERROR        | 7     | 35.0    |
+| INFO         | 6     | 30.0    |
+| WARN         | 4     | 20.0    |
+| DEBUG        | 3     | 15.0    |
++--------------+-------+---------+
+```
+
+## Example 7: Customizing the percent field name
+The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`:
+
+```ppl
+source=otellogs
+| top showperc=true percentfield='pct' severityText
+```
+
+The query returns the following results:
+
+```text
+fetched rows / total rows = 4/4
++--------------+-------+------+
+| severityText | count | pct  |
+|--------------+-------+------|
+| ERROR        | 7     | 35.0 |
+| INFO         | 6     | 30.0 |
+| WARN         | 4     | 20.0 |
+| DEBUG        | 3     | 15.0 |
++--------------+-------+------+
+```
+
+## Example 8: Specifying null value handling
 
 The following query specifies `usenull=false` to exclude null values:
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java

@@ -6,7 +6,9 @@
 package org.opensearch.sql.calcite.remote;
 
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES;
+import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
 import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows;
 import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder;
 
@@ -41,6 +43,43 @@ public void testTopCommandUseNullFalse() throws IOException {
     verifyNumOfRows(result, 5);
   }
 
+  @Test
+  public void testTopCommandShowPerc() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format("source=%s | top showperc=true age", TEST_INDEX_BANK_WITH_NULL_VALUES));
+    verifySchemaInOrder(
+        result, schema("age", "int"), schema("count", "bigint"), schema("percent", "double"));
+    verifyNumOfRows(result, 6);
+    verifyDataRows(
+        result,
+        rows(36, 2, 28.571429),
+        rows(28, 1, 14.285714),
+        rows(32, 1, 14.285714),
+        rows(33, 1, 14.285714),
+        rows(34, 1, 14.285714),
+        rows(null, 1, 14.285714));
+  }
+
+  @Test
+  public void testTopCommandShowPercWithoutShowCount() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | top showperc=true showcount=false age",
+                TEST_INDEX_BANK_WITH_NULL_VALUES));
+    verifySchemaInOrder(result, schema("age", "int"), schema("percent", "double"));
+    verifyNumOfRows(result, 6);
+    verifyDataRows(
+        result,
+        rows(36, 28.571429),
+        rows(28, 14.285714),
+        rows(32, 14.285714),
+        rows(33, 14.285714),
+        rows(34, 14.285714),
+        rows(null, 14.285714));
+  }
+
   @Test
   public void testTopCommandLegacyFalse() throws IOException {
     withSettings(

integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java

@@ -58,4 +58,20 @@ public void testTopNWithGroup() throws IOException {
       verifyDataRows(result, rows("F", "TX"), rows("M", "MD"));
     }
   }
+
+  @Test
+  public void testTopWithShowPerc() throws IOException {
+    JSONObject result =
+        executeQuery(String.format("source=%s | top showperc=true gender", TEST_INDEX_ACCOUNT));
+    if (isCalciteEnabled()) {
+      verifySchemaInOrder(
+          result,
+          schema("gender", "string"),
+          schema("count", "bigint"),
+          schema("percent", "double"));
+      verifyDataRows(result, rows("M", 507, 50.7), rows("F", 493, 49.3));
+    } else {
+      verifyDataRows(result, rows("M"), rows("F"));
+    }
+  }
 }

ppl/src/main/antlr/OpenSearchPPLLexer.g4

@@ -188,6 +188,8 @@ UNION:                              'UNION';
 MAXOUT:                             'MAXOUT';
 COUNTFIELD:                         'COUNTFIELD';
 SHOWCOUNT:                          'SHOWCOUNT';
+PERCENTFIELD:                       'PERCENTFIELD';
+SHOWPERC:                           'SHOWPERC';
 LIMIT:                              'LIMIT';
 USEOTHER:                           'USEOTHER';
 OTHERSTR:                           'OTHERSTR';

ppl/src/main/antlr/OpenSearchPPLParser.g4

@@ -489,6 +489,8 @@ rareTopCommand
 rareTopOption
    : COUNTFIELD EQUAL countField = stringLiteral
    | SHOWCOUNT EQUAL showCount = booleanLiteral
+   | PERCENTFIELD EQUAL percentField = stringLiteral
+   | SHOWPERC EQUAL showPerc = booleanLiteral
    | USENULL EQUAL useNull = booleanLiteral
    ;
 
@@ -1806,6 +1808,8 @@ searchableKeyWord
    | ANOMALY_SCORE_THRESHOLD
    | COUNTFIELD
    | SHOWCOUNT
+   | PERCENTFIELD
+   | SHOWPERC
    | MAXOUT
    | PATH
    | INPUT

ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java

@@ -307,6 +307,18 @@ public static List<Argument> getArgumentList(
         new Argument(
             RareTopN.Option.showCount.name(),
             opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE));
+    opt = ctx.rareTopOption().stream().filter(op -> op.percentField != null).findFirst();
+    list.add(
+        new Argument(
+            RareTopN.Option.percentField.name(),
+            opt.isPresent()
+                ? getArgumentValue(opt.get().percentField)
+                : new Literal("percent", DataType.STRING)));
+    opt = ctx.rareTopOption().stream().filter(op -> op.showPerc != null).findFirst();
+    list.add(
+        new Argument(
+            RareTopN.Option.showPerc.name(),
+            opt.isPresent() ? getArgumentValue(opt.get().showPerc) : Literal.FALSE));
     opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst();
     list.add(
         new Argument(

ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java

@@ -460,13 +460,16 @@ public String visitRareTopN(RareTopN node, String context) {
     Integer noOfResults = node.getNoOfResults();
     String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue();
     Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue();
+    String percentField = (String) arguments.get(RareTopN.Option.percentField.name()).getValue();
+    Boolean showPerc = (Boolean) arguments.get(RareTopN.Option.showPerc.name()).getValue();
     Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue();
     String fields = visitFieldList(node.getFields());
     String group = visitExpressionList(node.getGroupExprList());
     String options =
         UnresolvedPlanHelper.isCalciteEnabled(settings)
             ? StringUtils.format(
-                "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull)
+                "countfield='%s' showcount=%s percentfield='%s' showperc=%s usenull=%s ",
+                countField, showCount, percentField, showPerc, useNull)
             : "";
     return StringUtils.format(
         "%s | %s %d %s%s",