From 023d31fcb5210d1be9eebcc3be1cd1fb678c22b9 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Mon, 6 Apr 2026 13:59:08 -0700 Subject: [PATCH 1/3] [Feature] Add table function relation to SQL grammar for vectorSearch() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add table function relation support to the SQL parser: - New `tableFunctionRelation` alternative in `relation` grammar rule - Named argument syntax: `key=value` (e.g., table='index', field='vec') - Alias is required by grammar (FROM func(...) AS alias) - AstBuilder emits existing TableFunction + SubqueryAlias AST nodes - 3 parser unit tests: basic parse, with WHERE/ORDER BY/LIMIT, alias required This is a pure grammar change — no execution support yet. Queries will parse successfully but fail at the Analyzer with "unsupported function". Signed-off-by: Eric Wei --- sql/src/main/antlr/OpenSearchSQLParser.g4 | 9 +++ .../opensearch/sql/sql/parser/AstBuilder.java | 20 +++++++ .../sql/sql/parser/AstBuilderTest.java | 57 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5f7361160b3..3a41d4d8087 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -111,6 +111,15 @@ fromClause relation : tableName (AS? alias)? # tableAsRelation | LR_BRACKET subquery = querySpecification RR_BRACKET AS? alias # subqueryAsRelation + | qualifiedName LR_BRACKET tableFunctionArgs RR_BRACKET AS alias # tableFunctionRelation + ; + +tableFunctionArgs + : tableFunctionArg (COMMA tableFunctionArg)* + ; + +tableFunctionArg + : ident EQUAL_SYMBOL functionArg ; whereClause diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index bdbc360713c..aab0cc2dcff 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.SelectElementContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.SubqueryAsRelationContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.TableAsRelationContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.TableFunctionRelationContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.WhereClauseContext; import static org.opensearch.sql.sql.parser.ParserUtils.getTextInQuery; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; @@ -26,6 +27,7 @@ import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.DescribeRelation; import org.opensearch.sql.ast.tree.Filter; @@ -34,6 +36,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.SubqueryAlias; +import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -189,6 +192,23 @@ public UnresolvedPlan visitSubqueryAsRelation(SubqueryAsRelationContext ctx) { return new RelationSubquery(visit(ctx.subquery), subqueryAlias); } + @Override + public UnresolvedPlan visitTableFunctionRelation(TableFunctionRelationContext ctx) { + ImmutableList.Builder args = ImmutableList.builder(); + ctx.tableFunctionArgs() + .tableFunctionArg() + .forEach( + arg -> { + String argName = arg.ident().getText(); + UnresolvedExpression argValue = visitAstExpression(arg.functionArg()); + args.add(new UnresolvedArgument(argName, argValue)); + }); + TableFunction tableFunction = + new TableFunction(visitAstExpression(ctx.qualifiedName()), args.build()); + String alias = StringUtils.unquoteIdentifier(ctx.alias().getText()); + return new SubqueryAlias(alias, tableFunction); + } + @Override public UnresolvedPlan visitWhereClause(WhereClauseContext ctx) { return new Filter(visitAstExpression(ctx.expression())); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 1ecaa181e6f..cbd76706d66 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -7,7 +7,9 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; @@ -40,6 +42,10 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.NestedAllTupleFields; +import org.opensearch.sql.ast.expression.UnresolvedArgument; +import org.opensearch.sql.ast.tree.SubqueryAlias; +import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; class AstBuilderTest extends AstBuilderTestBase { @@ -131,6 +137,57 @@ public void can_build_from_index_with_alias_quoted() { buildAST("SELECT `t`.name FROM test `t` WHERE `t`.age = 30")); } + @Test + public void can_build_from_table_function() { + assertEquals( + project( + new SubqueryAlias( + "v", + new TableFunction( + qualifiedName("vectorSearch"), + ImmutableList.of( + new UnresolvedArgument("table", stringLiteral("products")), + new UnresolvedArgument("field", stringLiteral("embedding")), + new UnresolvedArgument("vector", stringLiteral("[0.1,0.2]")), + new UnresolvedArgument("option", stringLiteral("k=10"))))), + AllFields.of()), + buildAST( + "SELECT * FROM vectorSearch(" + + "table='products', field='embedding', " + + "vector='[0.1,0.2]', option='k=10') AS v")); + } + + @Test + public void can_build_from_table_function_with_where_and_order() { + // Verify parsing succeeds for table function with WHERE, ORDER BY, and LIMIT + UnresolvedPlan plan = + buildAST( + "SELECT s.title, s._score FROM vectorSearch(" + + "table='products', field='embedding', " + + "vector='[0.1,0.2]', option='k=10') AS s " + + "WHERE s.category = 'shoes' " + + "ORDER BY s._score DESC " + + "LIMIT 5"); + assertNotNull(plan); + // Verify the plan contains the expected structure + String planStr = plan.toString(); + assertTrue(planStr.contains("SubqueryAlias(alias=s")); + assertTrue(planStr.contains("TableFunction(functionName=vectorSearch")); + assertTrue(planStr.contains("UnresolvedArgument(argName=table, value=products)")); + assertTrue(planStr.contains("Filter(condition==(s.category, shoes)")); + } + + @Test + public void table_function_relation_requires_alias() { + assertThrows( + SyntaxCheckException.class, + () -> + buildAST( + "SELECT * FROM vectorSearch(" + + "table='products', field='embedding', " + + "vector='[0.1,0.2]', option='k=10')")); + } + @Test public void can_build_where_clause() { assertEquals( From 7b30898daab9997e2ef41af8221d2be281067108 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Mon, 6 Apr 2026 14:47:09 -0700 Subject: [PATCH 2/3] Address review feedback on table function relation grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Canonicalize argument names at parser boundary: unquoteIdentifier + toLowerCase(Locale.ROOT) in visitTableFunctionRelation so FIELD='x' and `field`='x' both produce argName="field" 2. Make AS keyword optional (AS? alias) for consistency with tableAsRelation and subqueryAsRelation grammar rules 3. Strengthen test coverage: - Full structural AST assertion for WHERE + ORDER BY + LIMIT (verifies Sort, Limit, Filter nodes, not just toString) - Argument reorder test proves names resolve by name not position - Case canonicalization test (TABLE= → table=) - Alias-without-AS test (FROM func(...) v) Signed-off-by: Eric Wei --- sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 +- .../opensearch/sql/sql/parser/AstBuilder.java | 5 +- .../sql/sql/parser/AstBuilderTest.java | 96 ++++++++++++++++--- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 3a41d4d8087..68887b0f670 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -111,7 +111,7 @@ fromClause relation : tableName (AS? alias)? # tableAsRelation | LR_BRACKET subquery = querySpecification RR_BRACKET AS? alias # subqueryAsRelation - | qualifiedName LR_BRACKET tableFunctionArgs RR_BRACKET AS alias # tableFunctionRelation + | qualifiedName LR_BRACKET tableFunctionArgs RR_BRACKET AS? alias # tableFunctionRelation ; tableFunctionArgs diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index aab0cc2dcff..2134a46e681 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import java.util.Collections; +import java.util.Locale; import java.util.Optional; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; @@ -199,7 +200,9 @@ public UnresolvedPlan visitTableFunctionRelation(TableFunctionRelationContext ct .tableFunctionArg() .forEach( arg -> { - String argName = arg.ident().getText(); + String argName = + StringUtils.unquoteIdentifier(arg.ident().getText()) + .toLowerCase(Locale.ROOT); UnresolvedExpression argValue = visitAstExpression(arg.functionArg()); args.add(new UnresolvedArgument(argName, argValue)); }); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index cbd76706d66..a74b87c856d 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -7,9 +7,7 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; @@ -45,7 +43,6 @@ import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; -import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; class AstBuilderTest extends AstBuilderTestBase { @@ -158,23 +155,94 @@ public void can_build_from_table_function() { } @Test - public void can_build_from_table_function_with_where_and_order() { - // Verify parsing succeeds for table function with WHERE, ORDER BY, and LIMIT - UnresolvedPlan plan = + public void can_build_from_table_function_with_where_order_limit() { + assertEquals( + project( + limit( + sort( + filter( + new SubqueryAlias( + "s", + new TableFunction( + qualifiedName("vectorSearch"), + ImmutableList.of( + new UnresolvedArgument("table", stringLiteral("products")), + new UnresolvedArgument("field", stringLiteral("embedding")), + new UnresolvedArgument( + "vector", stringLiteral("[0.1,0.2]")), + new UnresolvedArgument( + "option", stringLiteral("k=10"))))), + function("=", qualifiedName("s", "category"), stringLiteral("shoes"))), + field( + qualifiedName("s", "_score"), + argument("asc", booleanLiteral(false)))), + 5, + 0), + alias("s.title", qualifiedName("s", "title")), + alias("s._score", qualifiedName("s", "_score"))), buildAST( "SELECT s.title, s._score FROM vectorSearch(" + "table='products', field='embedding', " + "vector='[0.1,0.2]', option='k=10') AS s " + "WHERE s.category = 'shoes' " + "ORDER BY s._score DESC " - + "LIMIT 5"); - assertNotNull(plan); - // Verify the plan contains the expected structure - String planStr = plan.toString(); - assertTrue(planStr.contains("SubqueryAlias(alias=s")); - assertTrue(planStr.contains("TableFunction(functionName=vectorSearch")); - assertTrue(planStr.contains("UnresolvedArgument(argName=table, value=products)")); - assertTrue(planStr.contains("Filter(condition==(s.category, shoes)")); + + "LIMIT 5")); + } + + @Test + public void table_function_args_are_resolved_by_name_not_position() { + assertEquals( + project( + new SubqueryAlias( + "v", + new TableFunction( + qualifiedName("vectorSearch"), + ImmutableList.of( + new UnresolvedArgument("option", stringLiteral("k=10")), + new UnresolvedArgument("field", stringLiteral("embedding")), + new UnresolvedArgument("table", stringLiteral("products")), + new UnresolvedArgument("vector", stringLiteral("[0.1,0.2]"))))), + AllFields.of()), + buildAST( + "SELECT * FROM vectorSearch(" + + "option='k=10', field='embedding', " + + "table='products', vector='[0.1,0.2]') AS v")); + } + + @Test + public void table_function_arg_names_are_canonicalized() { + assertEquals( + project( + new SubqueryAlias( + "v", + new TableFunction( + qualifiedName("vectorSearch"), + ImmutableList.of( + new UnresolvedArgument("table", stringLiteral("products")), + new UnresolvedArgument("field", stringLiteral("embedding")), + new UnresolvedArgument("vector", stringLiteral("[0.1,0.2]")), + new UnresolvedArgument("option", stringLiteral("k=10"))))), + AllFields.of()), + buildAST( + "SELECT * FROM vectorSearch(" + + "TABLE='products', FIELD='embedding', " + + "VECTOR='[0.1,0.2]', OPTION='k=10') AS v")); + } + + @Test + public void table_function_allows_alias_without_as_keyword() { + assertEquals( + project( + new SubqueryAlias( + "v", + new TableFunction( + qualifiedName("vectorSearch"), + ImmutableList.of( + new UnresolvedArgument("table", stringLiteral("products")), + new UnresolvedArgument("vector", stringLiteral("[0.1]"))))), + AllFields.of()), + buildAST( + "SELECT * FROM vectorSearch(table='products', vector='[0.1]') v")); } @Test From d73e73ff27206fa2d913f4b044ab967d36de69c2 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Mon, 6 Apr 2026 14:53:52 -0700 Subject: [PATCH 3/3] Apply spotless formatting Signed-off-by: Eric Wei --- .../org/opensearch/sql/sql/parser/AstBuilder.java | 3 +-- .../opensearch/sql/sql/parser/AstBuilderTest.java | 13 ++++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index 2134a46e681..bacdf9d7378 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -201,8 +201,7 @@ public UnresolvedPlan visitTableFunctionRelation(TableFunctionRelationContext ct .forEach( arg -> { String argName = - StringUtils.unquoteIdentifier(arg.ident().getText()) - .toLowerCase(Locale.ROOT); + StringUtils.unquoteIdentifier(arg.ident().getText()).toLowerCase(Locale.ROOT); UnresolvedExpression argValue = visitAstExpression(arg.functionArg()); args.add(new UnresolvedArgument(argName, argValue)); }); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index a74b87c856d..da48b9bb7fc 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -168,14 +168,10 @@ public void can_build_from_table_function_with_where_order_limit() { ImmutableList.of( new UnresolvedArgument("table", stringLiteral("products")), new UnresolvedArgument("field", stringLiteral("embedding")), - new UnresolvedArgument( - "vector", stringLiteral("[0.1,0.2]")), - new UnresolvedArgument( - "option", stringLiteral("k=10"))))), + new UnresolvedArgument("vector", stringLiteral("[0.1,0.2]")), + new UnresolvedArgument("option", stringLiteral("k=10"))))), function("=", qualifiedName("s", "category"), stringLiteral("shoes"))), - field( - qualifiedName("s", "_score"), - argument("asc", booleanLiteral(false)))), + field(qualifiedName("s", "_score"), argument("asc", booleanLiteral(false)))), 5, 0), alias("s.title", qualifiedName("s", "title")), @@ -241,8 +237,7 @@ public void table_function_allows_alias_without_as_keyword() { new UnresolvedArgument("table", stringLiteral("products")), new UnresolvedArgument("vector", stringLiteral("[0.1]"))))), AllFields.of()), - buildAST( - "SELECT * FROM vectorSearch(table='products', vector='[0.1]') v")); + buildAST("SELECT * FROM vectorSearch(table='products', vector='[0.1]') v")); } @Test