diff --git a/pom.xml b/pom.xml index f979056f76891..c792fc783d343 100644 --- a/pom.xml +++ b/pom.xml @@ -221,6 +221,7 @@ presto-function-server presto-router-example-plugin-scheduler presto-plan-checker-router-plugin + presto-sql-invoked-functions-plugin presto-spark-classloader-spark${dep.pos.classloader.module-name.suffix} diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 0e6cdf23cb5b5..4b855fb5df084 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -494,6 +494,12 @@ + + com.facebook.presto + presto-sql-invoked-functions-plugin + ${project.version} + test + diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestDistributedQueriesSingleNode.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestDistributedQueriesSingleNode.java index bfaec5f22852d..e3328a940c71b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestDistributedQueriesSingleNode.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestDistributedQueriesSingleNode.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; @@ -32,11 +33,13 @@ protected QueryRunner createQueryRunner() { ImmutableMap.Builder coordinatorProperties = ImmutableMap.builder(); coordinatorProperties.put("single-node-execution-enabled", "true"); - return HiveQueryRunner.createQueryRunner( + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner( getTables(), ImmutableMap.of(), coordinatorProperties.build(), Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedNanQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedNanQueries.java index fa2738c63fd72..fc74dbb2f4433 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedNanQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedNanQueries.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestNanQueries; import com.google.common.collect.ImmutableList; @@ -28,6 +29,9 @@ public class TestHiveDistributedNanQueries protected QueryRunner createQueryRunner() throws Exception { - return HiveQueryRunner.createQueryRunner(ImmutableList.of(), ImmutableMap.of("use-new-nan-definition", "true"), ImmutableMap.of(), Optional.empty()); + QueryRunner queryRunner = + HiveQueryRunner.createQueryRunner(ImmutableList.of(), ImmutableMap.of("use-new-nan-definition", "true"), ImmutableMap.of(), Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueries.java index 7552535ae9a4f..134cf4b101f1f 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueries.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.hive.TestHiveEventListenerPlugin.TestingHiveEventListener; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.testing.MaterializedResult; @@ -59,7 +60,9 @@ public class TestHiveDistributedQueries protected QueryRunner createQueryRunner() throws Exception { - return HiveQueryRunner.createQueryRunner(getTables()); + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(getTables()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java index 226bbf3888b53..a878243e32b91 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; @@ -40,7 +41,9 @@ public class TestHiveDistributedQueriesWithExchangeMaterialization protected QueryRunner createQueryRunner() throws Exception { - return createMaterializingQueryRunner(getTables()); + QueryRunner queryRunner = createMaterializingQueryRunner(getTables()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Test diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java index d0bf8f6499428..2477f9c31cf46 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; @@ -30,13 +31,15 @@ public class TestHiveDistributedQueriesWithOptimizedRepartitioning protected QueryRunner createQueryRunner() throws Exception { - return HiveQueryRunner.createQueryRunner( + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner( getTables(), ImmutableMap.of( "experimental.optimized-repartitioning", "true", // Use small SerializedPages to force flushing "driver.max-page-partitioning-buffer-size", "10000B"), Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithThriftRpc.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithThriftRpc.java index 6f877f35a4ce3..1bbc9590dcd50 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithThriftRpc.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithThriftRpc.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; @@ -30,13 +31,15 @@ public class TestHiveDistributedQueriesWithThriftRpc protected QueryRunner createQueryRunner() throws Exception { - return HiveQueryRunner.createQueryRunner( + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner( getTables(), ImmutableMap.of( "internal-communication.task-communication-protocol", "THRIFT", "internal-communication.server-info-communication-protocol", "THRIFT"), ImmutableMap.of(), Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java index 28ec3c9ef3491..dc816175ffb12 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; @@ -34,7 +35,7 @@ public class TestHivePushdownDistributedQueries protected QueryRunner createQueryRunner() throws Exception { - return HiveQueryRunner.createQueryRunner( + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner( getTables(), ImmutableMap.of("experimental.pushdown-subfields-enabled", "true", "experimental.pushdown-dereference-enabled", "true"), @@ -44,6 +45,8 @@ protected QueryRunner createQueryRunner() "hive.partial_aggregation_pushdown_enabled", "true", "hive.partial_aggregation_pushdown_for_variable_length_datatypes_enabled", "true"), Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestLambdaSubfieldPruning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestLambdaSubfieldPruning.java index a0559b03b0302..b99eff80743ab 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestLambdaSubfieldPruning.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestLambdaSubfieldPruning.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.DistributedQueryRunner; @@ -126,6 +127,7 @@ private static DistributedQueryRunner createLineItemExTable(DistributedQueryRunn "FROM lineitem \n"); } + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); return queryRunner; } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java index d8bb271cebcd0..0aef10a5b4bf0 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.Session; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestDistributedQueries; @@ -46,7 +47,7 @@ protected QueryRunner createQueryRunner() .put("hive.partial_aggregation_pushdown_enabled", "true") .put("hive.partial_aggregation_pushdown_for_variable_length_datatypes_enabled", "true") .build(); - return HiveQueryRunner.createQueryRunner( + QueryRunner queryRunner = HiveQueryRunner.createQueryRunner( getTables(), ImmutableMap.of( "experimental.pushdown-subfields-enabled", "true", @@ -54,6 +55,8 @@ protected QueryRunner createQueryRunner() "sql-standard", parquetProperties, Optional.empty()); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Test diff --git a/presto-i18n-functions/src/test/java/com/facebook/presto/i18n/functions/TestMyanmarFunctions.java b/presto-i18n-functions/src/test/java/com/facebook/presto/i18n/functions/TestMyanmarFunctions.java index 1f5bc00908408..f0259c46f8c5c 100644 --- a/presto-i18n-functions/src/test/java/com/facebook/presto/i18n/functions/TestMyanmarFunctions.java +++ b/presto-i18n-functions/src/test/java/com/facebook/presto/i18n/functions/TestMyanmarFunctions.java @@ -14,15 +14,23 @@ package com.facebook.presto.i18n.functions; import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; public class TestMyanmarFunctions extends AbstractTestFunctions { + public TestMyanmarFunctions() + { + super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false); + } + @BeforeClass public void setUp() { diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index c51c38e73f2f7..b9ff6288fe744 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -461,9 +461,18 @@ io.netty netty-transport + com.facebook.presto presto-built-in-worker-function-tools + ${project.version} + test + + + + com.facebook.presto + presto-sql-invoked-functions-plugin + ${project.version} test diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java index 964b0e9a0891c..dcd00723dc890 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java @@ -33,8 +33,6 @@ import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -49,7 +47,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.HOURS; @@ -58,8 +55,6 @@ public abstract class BuiltInSpecialFunctionNamespaceManager { protected volatile FunctionMap functions = new FunctionMap(); private final FunctionAndTypeManager functionAndTypeManager; - protected final Supplier cachedFunctions = - Suppliers.memoize(this::createFunctionMap); private final LoadingCache specializedFunctionKeyCache; private final LoadingCache specializedScalarCache; @@ -85,11 +80,7 @@ public BuiltInSpecialFunctionNamespaceManager(FunctionAndTypeManager functionAnd @Override public Collection getFunctions(Optional transactionHandle, QualifiedObjectName functionName) { - if (functions.list().isEmpty() || - (!functionName.getCatalogSchemaName().equals(functionAndTypeManager.getDefaultNamespace()))) { - return emptyList(); - } - return cachedFunctions.get().get(functionName); + return functions.get(functionName); } /** @@ -98,7 +89,7 @@ public Collection getFunctions(Optional listFunctions(Optional likePattern, Optional escape) { - return cachedFunctions.get().list(); + return functions.list(); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index 1c3afcf3fa115..40c493fc887c6 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -203,11 +203,6 @@ import com.facebook.presto.operator.scalar.WilsonInterval; import com.facebook.presto.operator.scalar.WordStemFunction; import com.facebook.presto.operator.scalar.queryplan.JsonPrestoQueryPlanFunctions; -import com.facebook.presto.operator.scalar.sql.ArraySqlFunctions; -import com.facebook.presto.operator.scalar.sql.MapNormalizeFunction; -import com.facebook.presto.operator.scalar.sql.MapSqlFunctions; -import com.facebook.presto.operator.scalar.sql.SimpleSamplingPercent; -import com.facebook.presto.operator.scalar.sql.StringSqlFunctions; import com.facebook.presto.operator.window.CumulativeDistributionFunction; import com.facebook.presto.operator.window.DenseRankFunction; import com.facebook.presto.operator.window.FirstValueFunction; @@ -994,12 +989,6 @@ private List getBuiltInFunctions(FunctionsConfig function .aggregate(ThetaSketchAggregationFunction.class) .scalars(ThetaSketchFunctions.class) .function(MergeTDigestFunction.MERGE) - .sqlInvokedScalar(MapNormalizeFunction.class) - .sqlInvokedScalars(ArraySqlFunctions.class) - .sqlInvokedScalars(ArrayIntersectFunction.class) - .sqlInvokedScalars(MapSqlFunctions.class) - .sqlInvokedScalars(SimpleSamplingPercent.class) - .sqlInvokedScalars(StringSqlFunctions.class) .scalar(DynamicFilterPlaceholderFunction.class) .scalars(EnumCasts.class) .scalars(LongEnumOperators.class) diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java index f3dab976902a9..799dd5f8074fe 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java @@ -15,7 +15,6 @@ import com.facebook.presto.operator.scalar.annotations.CodegenScalarFromAnnotationsParser; import com.facebook.presto.operator.scalar.annotations.ScalarFromAnnotationsParser; -import com.facebook.presto.operator.scalar.annotations.SqlInvokedScalarFromAnnotationsParser; import com.facebook.presto.operator.window.WindowAnnotationsParser; import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.function.WindowFunction; @@ -24,7 +23,6 @@ import java.util.ArrayList; import java.util.List; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static java.util.Objects.requireNonNull; public class FunctionListBuilder @@ -61,18 +59,6 @@ public FunctionListBuilder scalars(Class clazz) return this; } - public FunctionListBuilder sqlInvokedScalar(Class clazz) - { - functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz, JAVA_BUILTIN_NAMESPACE)); - return this; - } - - public FunctionListBuilder sqlInvokedScalars(Class clazz) - { - functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE)); - return this; - } - public FunctionListBuilder codegenScalars(Class clazz) { functions.addAll(CodegenScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)); diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java index d977420d28339..4ee510a86c467 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java @@ -19,8 +19,6 @@ import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.OperatorDependency; import com.facebook.presto.spi.function.ScalarFunction; -import com.facebook.presto.spi.function.SqlInvokedScalarFunction; -import com.facebook.presto.spi.function.SqlParameter; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; @@ -60,14 +58,4 @@ public static Block intersect( return typedSet.getBlock(); } - - @SqlInvokedScalarFunction(value = "array_intersect", deterministic = true, calledOnNullInput = false) - @Description("Intersects elements of all arrays in the given array") - @TypeParameter("T") - @SqlParameter(name = "input", type = "array>") - @SqlType("array") - public static String arrayIntersectArray() - { - return "RETURN reduce(input, IF((cardinality(input) = 0), ARRAY[], input[1]), (s, x) -> array_intersect(s, x), (s) -> s)"; - } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/QueryRunner.java b/presto-main-base/src/main/java/com/facebook/presto/testing/QueryRunner.java index 5ab9fcbb94a71..f6c1a0b9b1405 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/testing/QueryRunner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/QueryRunner.java @@ -126,6 +126,11 @@ default void loadPlanCheckerProviderManager(String planCheckerProviderName, Map< throw new UnsupportedOperationException(); } + default void triggerConflictCheckWithBuiltInFunctions() + { + throw new UnsupportedOperationException(); + } + class MaterializedResultWithPlan { private final MaterializedResult materializedResult; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index d31ae0b09eb02..0644a6ebbdb68 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -59,6 +59,7 @@ public abstract class AbstractTestFunctions private final FeaturesConfig featuresConfig; private final FunctionsConfig functionsConfig; protected FunctionAssertions functionAssertions; + private final boolean loadInlinedSqlInvokedFunctionsPlugin; protected AbstractTestFunctions() { @@ -81,18 +82,23 @@ protected AbstractTestFunctions(FunctionsConfig functionsConfig) } protected AbstractTestFunctions(Session session, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig) + { + this(session, featuresConfig, functionsConfig, true); + } + protected AbstractTestFunctions(Session session, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, boolean loadInlinedSqlInvokedFunctionsPlugin) { this.session = requireNonNull(session, "session is null"); this.featuresConfig = requireNonNull(featuresConfig, "featuresConfig is null"); this.functionsConfig = requireNonNull(functionsConfig, "config is null") .setLegacyLogFunction(true) .setUseNewNanDefinition(true); + this.loadInlinedSqlInvokedFunctionsPlugin = loadInlinedSqlInvokedFunctionsPlugin; } @BeforeClass public final void initTestFunctions() { - functionAssertions = new FunctionAssertions(session, featuresConfig, functionsConfig, false); + functionAssertions = new FunctionAssertions(session, featuresConfig, functionsConfig, false, loadInlinedSqlInvokedFunctionsPlugin); } @AfterClass(alwaysRun = true) diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java index 17ba3c70823db..b2965a54329c5 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java @@ -41,6 +41,7 @@ import com.facebook.presto.operator.project.CursorProcessor; import com.facebook.presto.operator.project.PageProcessor; import com.facebook.presto.operator.project.PageProjectionWithOutputs; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; @@ -221,20 +222,25 @@ public FunctionAssertions() public FunctionAssertions(Session session) { - this(session, new FeaturesConfig(), new FunctionsConfig(), false); + this(session, new FeaturesConfig(), new FunctionsConfig(), false, true); } public FunctionAssertions(Session session, FeaturesConfig featuresConfig) { - this(session, featuresConfig, new FunctionsConfig(), false); + this(session, featuresConfig, new FunctionsConfig(), false, true); } public FunctionAssertions(Session session, FunctionsConfig functionsConfig) { - this(session, new FeaturesConfig(), functionsConfig, false); + this(session, new FeaturesConfig(), functionsConfig, false, true); } public FunctionAssertions(Session session, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, boolean refreshSession) + { + this(session, featuresConfig, functionsConfig, refreshSession, true); + } + + public FunctionAssertions(Session session, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, boolean refreshSession, boolean loadInlinedSqlInvokedFunctionsPlugin) { requireNonNull(session, "session is null"); runner = new LocalQueryRunner(session, featuresConfig, functionsConfig); @@ -244,6 +250,9 @@ public FunctionAssertions(Session session, FeaturesConfig featuresConfig, Functi else { this.session = session; } + if (loadInlinedSqlInvokedFunctionsPlugin) { + runner.installPlugin(new SqlInvokedFunctionsPlugin()); + } metadata = runner.getMetadata(); compiler = runner.getExpressionCompiler(); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java index 4bd3f167ac11a..194cf489d3998 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestCrossJoinWithArrayNotContainsToAntiJoin.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.JoinNode; @@ -22,9 +23,11 @@ import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import static com.facebook.presto.SystemSessionProperties.REWRITE_CROSS_JOIN_ARRAY_NOT_CONTAINS_TO_ANTI_JOIN; @@ -33,10 +36,18 @@ import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; +import static java.util.Collections.singletonList; public class TestCrossJoinWithArrayNotContainsToAntiJoin extends BaseRuleTest { + @BeforeClass + @Override + public void setUp() + { + tester = new RuleTester(singletonList(new SqlInvokedFunctionsPlugin())); + } + @Test public void testTriggerForBigInt() { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java index 657093aec5be2..cdf060f86d390 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLeftJoinWithArrayContainsToEquiJoinCondition.java @@ -14,11 +14,14 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.plan.JoinType; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Optional; @@ -32,10 +35,18 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.unnest; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static java.util.Collections.singletonList; public class TestLeftJoinWithArrayContainsToEquiJoinCondition extends BaseRuleTest { + @BeforeClass + @Override + public void setUp() + { + tester = new RuleTester(singletonList(new SqlInvokedFunctionsPlugin())); + } + @Test public void testTriggerForBigIntArrayRightSide() { diff --git a/presto-main/etc/config.properties b/presto-main/etc/config.properties index a120422c4b6f1..164015db096d4 100644 --- a/presto-main/etc/config.properties +++ b/presto-main/etc/config.properties @@ -51,7 +51,8 @@ plugin.bundles=\ ../presto-node-ttl-fetchers/pom.xml,\ ../presto-hive-function-namespace/pom.xml,\ ../presto-delta/pom.xml,\ - ../presto-hudi/pom.xml + ../presto-hudi/pom.xml, \ + ../presto-sql-invoked-functions-plugin/pom.xml presto.version=testversion node-scheduler.include-coordinator=true diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java b/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java index dffbefd9189aa..4ee697957e540 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java @@ -15,13 +15,21 @@ package com.facebook.presto.ml; import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; import org.testng.annotations.BeforeClass; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; abstract class AbstractTestMLFunctions extends AbstractTestFunctions { + public AbstractTestMLFunctions() + { + super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false); + } + @BeforeClass protected void registerFunctions() { diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index 8f849f866335a..4f48109d76937 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -328,6 +328,19 @@ presto-jdbc test + + + org.apache.commons + commons-lang3 + test + + + + com.facebook.presto + presto-sql-invoked-functions-plugin + ${project.version} + test + diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java index bfa21641206b2..f45f20c545d00 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeArrayFunctionQueries.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.nativeworker; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; @@ -20,18 +21,24 @@ public class TestPrestoNativeArrayFunctionQueries extends AbstractTestNativeArrayFunctionQueries { @Override - protected QueryRunner createQueryRunner() throws Exception + protected QueryRunner createQueryRunner() + throws Exception { - return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override - protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception { - return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java index 4b1d27f30426e..bcba83d00da65 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesJSON.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.nativeworker; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; @@ -23,17 +24,21 @@ public class TestPrestoNativeGeneralQueriesJSON protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java index a1c33cd09f51c..fd82dcde9209e 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeGeneralQueriesThrift.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.nativeworker; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; @@ -23,18 +24,22 @@ public class TestPrestoNativeGeneralQueriesThrift protected QueryRunner createQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .setUseThrift(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } @Override protected ExpectedQueryRunner createExpectedQueryRunner() throws Exception { - return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + QueryRunner queryRunner = PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java index 71c884e828554..b54a9711940b9 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java @@ -123,4 +123,16 @@ public void testKeyBasedSampling() public void testDefaultSamplingPercent() { } + + @Override + @Test(enabled = false) + public void testLeftJoinWithArrayContainsCondition() + { + } + + @Override + @Test(enabled = false) + public void testTry() + { + } } diff --git a/presto-pinot/src/test/java/com/facebook/presto/pinot/udf/TestPinotFunctions.java b/presto-pinot/src/test/java/com/facebook/presto/pinot/udf/TestPinotFunctions.java index acfd681d0e48b..3dec94b4fe4ce 100644 --- a/presto-pinot/src/test/java/com/facebook/presto/pinot/udf/TestPinotFunctions.java +++ b/presto-pinot/src/test/java/com/facebook/presto/pinot/udf/TestPinotFunctions.java @@ -15,15 +15,23 @@ import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.pinot.PinotPlugin; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; public class TestPinotFunctions extends AbstractTestFunctions { + public TestPinotFunctions() + { + super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false); + } + @BeforeClass public void setUp() { diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 14bcfd5b3c31a..10359099bce2a 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -280,4 +280,10 @@ + + + + + + diff --git a/presto-sql-invoked-functions-plugin/pom.xml b/presto-sql-invoked-functions-plugin/pom.xml new file mode 100644 index 0000000000000..5ab338547b4ef --- /dev/null +++ b/presto-sql-invoked-functions-plugin/pom.xml @@ -0,0 +1,29 @@ + + 4.0.0 + + com.facebook.presto + presto-root + 0.295-SNAPSHOT + + + presto-sql-invoked-functions-plugin + Presto - Sql invoked functions plugin + presto-plugin + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + provided + + + com.google.guava + guava + + + diff --git a/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArrayIntersectFunction.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArrayIntersectFunction.java new file mode 100644 index 0000000000000..860faede7942d --- /dev/null +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArrayIntersectFunction.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.scalar.sql; + +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.SqlInvokedScalarFunction; +import com.facebook.presto.spi.function.SqlParameter; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; + +public class ArrayIntersectFunction +{ + private ArrayIntersectFunction() {} + + @SqlInvokedScalarFunction(value = "array_intersect", deterministic = true, calledOnNullInput = false) + @Description("Intersects elements of all arrays in the given array") + @TypeParameter("T") + @SqlParameter(name = "input", type = "array>") + @SqlType("array") + public static String arrayIntersectArray() + { + return "RETURN reduce(input, IF((cardinality(input) = 0), ARRAY[], input[1]), (s, x) -> array_intersect(s, x), (s) -> s)"; + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/ArraySqlFunctions.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArraySqlFunctions.java similarity index 99% rename from presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/ArraySqlFunctions.java rename to presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArraySqlFunctions.java index 2fc1218ea2408..b5ab56eb6f312 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/ArraySqlFunctions.java +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/ArraySqlFunctions.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator.scalar.sql; +package com.facebook.presto.scalar.sql; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapNormalizeFunction.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapNormalizeFunction.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapNormalizeFunction.java rename to presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapNormalizeFunction.java index 1618dac2c4356..99104f3027b8d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapNormalizeFunction.java +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapNormalizeFunction.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator.scalar.sql; +package com.facebook.presto.scalar.sql; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapSqlFunctions.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapSqlFunctions.java similarity index 99% rename from presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapSqlFunctions.java rename to presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapSqlFunctions.java index 0cf9558a22989..498c068dedf39 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/MapSqlFunctions.java +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/MapSqlFunctions.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator.scalar.sql; +package com.facebook.presto.scalar.sql; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/SimpleSamplingPercent.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SimpleSamplingPercent.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/SimpleSamplingPercent.java rename to presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SimpleSamplingPercent.java index 473bbedafd637..5aa6b6b5d966e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/SimpleSamplingPercent.java +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SimpleSamplingPercent.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator.scalar.sql; +package com.facebook.presto.scalar.sql; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; diff --git a/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SqlInvokedFunctionsPlugin.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SqlInvokedFunctionsPlugin.java new file mode 100644 index 0000000000000..9a1e3650f72fe --- /dev/null +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/SqlInvokedFunctionsPlugin.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.scalar.sql; + +import com.facebook.presto.spi.Plugin; +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +public class SqlInvokedFunctionsPlugin + implements Plugin +{ + @Override + public Set> getSqlInvokedFunctions() + { + return ImmutableSet.>builder() + .add(ArraySqlFunctions.class) + .add(MapNormalizeFunction.class) + .add(MapSqlFunctions.class) + .add(SimpleSamplingPercent.class) + .add(StringSqlFunctions.class) + .add(ArrayIntersectFunction.class) + .build(); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/StringSqlFunctions.java b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/StringSqlFunctions.java similarity index 97% rename from presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/StringSqlFunctions.java rename to presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/StringSqlFunctions.java index 1b0e6e15661ed..9201e4bb1cf94 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/sql/StringSqlFunctions.java +++ b/presto-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/StringSqlFunctions.java @@ -12,7 +12,7 @@ * limitations under the License. */ -package com.facebook.presto.operator.scalar.sql; +package com.facebook.presto.scalar.sql; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.SqlInvokedScalarFunction; diff --git a/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataDateFunctions.java b/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataDateFunctions.java index 9b4a72aacc792..16bd47af0d5c1 100644 --- a/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataDateFunctions.java +++ b/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataDateFunctions.java @@ -18,6 +18,8 @@ import com.facebook.presto.common.type.SqlDate; import com.facebook.presto.common.type.TimestampType; import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -40,7 +42,7 @@ public class TestTeradataDateFunctions protected TestTeradataDateFunctions() { - super(SESSION); + super(SESSION, new FeaturesConfig(), new FunctionsConfig(), false); } @BeforeClass diff --git a/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataFunctions.java b/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataFunctions.java index be6efbffec1ef..e724ca94566e7 100644 --- a/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataFunctions.java +++ b/presto-teradata-functions/src/test/java/com/facebook/presto/teradata/functions/TestTeradataFunctions.java @@ -14,9 +14,12 @@ package com.facebook.presto.teradata.functions; import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.FunctionsConfig; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.common.type.VarcharType.createVarcharType; @@ -25,6 +28,11 @@ public class TestTeradataFunctions extends AbstractTestFunctions { + public TestTeradataFunctions() + { + super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false); + } + @BeforeClass public void setUp() { diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 3623dd0991913..963a82a22adad 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -378,6 +378,13 @@ ratis-metrics-default test + + + com.facebook.presto + presto-sql-invoked-functions-plugin + ${project.version} + test + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 67bf65f67eb55..6949fc4e9c8b3 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -1040,6 +1040,14 @@ public void loadPlanCheckerProviderManager(String planCheckerProviderName, Map> getSqlInvokedFunctions() .add(TestDuplicateSqlInvokedFunctions.class) .build(); } + + @Override + public Set> getFunctions() + { + return ImmutableSet.>builder() + .add(TestFunctions.class) + // Adding a SQL Invoked function in the built-in functions to mimic a conflict. + .add(ArrayIntersectFunction.class) + .build(); + } } // As soon as we trigger the conflict check with the built-in functions, an error will be thrown if duplicate signatures are found. diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java index b18173ff0bf20..8d6910ccaa59d 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; @@ -23,6 +24,8 @@ public class TestDistributedEngineOnlyQueries protected QueryRunner createQueryRunner() throws Exception { - return TpchQueryRunnerBuilder.builder().build(); + QueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java index e945493c2969e..1ab8522eb50a3 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; @@ -63,6 +64,7 @@ public static QueryRunner localCreateQueryRunner() try { queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); return queryRunner; } catch (Exception e) { diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueriesWithTempStorage.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueriesWithTempStorage.java index c4ca56a3f78c0..739c5dc826631 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueriesWithTempStorage.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueriesWithTempStorage.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; @@ -57,6 +58,8 @@ public static DistributedQueryRunner localCreateQueryRunner() try { queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; } catch (Exception e) { diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java index b381b01f5ada3..e7f2cf5a9a781 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.CatalogSchemaTableName; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.ColumnConstraint; @@ -78,6 +79,8 @@ public static LocalQueryRunner createLocalQueryRunner() sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); sessionPropertyManager.addConnectorSessionProperties(new ConnectorId(TESTING_CATALOG), TEST_CATALOG_PROPERTIES); + localQueryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return localQueryRunner; } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java index 98fdee5fe7ebf..8fbcc731fec0a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.common.type.Type; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.testing.MaterializedResult; @@ -74,6 +75,7 @@ protected QueryRunner createQueryRunner() sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); sessionPropertyManager.addConnectorSessionProperties(new ConnectorId(TESTING_CATALOG), TEST_CATALOG_PROPERTIES); + localQueryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); return localQueryRunner; } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java index cafb8e59e7fc1..e4ec0ad5e39a8 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; import com.facebook.presto.common.type.UserDefinedType; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.function.Parameter; import com.facebook.presto.spi.function.RoutineCharacteristics; import com.facebook.presto.spi.function.SqlFunctionId; @@ -99,7 +100,7 @@ protected QueryRunner createQueryRunner() queryRunner.getMetadata().getFunctionAndTypeManager().addUserDefinedType(COUNTRY_ENUM); queryRunner.execute("CREATE TYPE testing.type.person AS (first_name varchar, last_name varchar, age tinyint, country testing.enum.country)"); - + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); return queryRunner; } catch (Exception e) { diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java index 3cb9f26f1b8fd..add9d0be8cd07 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestVerboseOptimizerInfo.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.testing.MaterializedResult; @@ -71,7 +72,7 @@ public static LocalQueryRunner createLocalQueryRunner() SessionPropertyManager sessionPropertyManager = localQueryRunner.getMetadata().getSessionPropertyManager(); sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); sessionPropertyManager.addConnectorSessionProperties(new ConnectorId(TESTING_CATALOG), TEST_CATALOG_PROPERTIES); - + localQueryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); return localQueryRunner; }