diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index c6d77e28e265..ba8c773c977c 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1167,8 +1167,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("detect escaped path and report the migration guide") .exclude("ignore the escaped path check when the flag is off") .excludeByPrefix("SPARK-51187") - // Rewrite for the query plan check - .excludeByPrefix("SPARK-49905") enableSuite[GlutenQueryExecutionSuite] // Rewritten to set root logger level to INFO so that logs can be parsed .exclude("Logging plan changes for execution") @@ -1181,45 +1179,49 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLMetricsSuite] enableSuite[GlutenAcceptsLatestSeenOffsetSuite] enableSuite[GlutenCommitLogSuite] - // TODO: 4.x enableSuite[GlutenEventTimeWatermarkSuite] + enableSuite[GlutenEventTimeWatermarkSuite] enableSuite[GlutenFileStreamSinkV1Suite] - // TODO: 4.x enableSuite[GlutenFileStreamSinkV2Suite] // 1 failure + enableSuite[GlutenFileStreamSinkV2Suite] enableSuite[GlutenFileStreamSourceStressTestSuite] - // TODO: 4.x enableSuite[GlutenFileStreamSourceSuite] + enableSuite[GlutenFileStreamSourceSuite] enableSuite[GlutenFileStreamStressSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] // failures with GlutenPlugin + enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] enableSuite[GlutenFlatMapGroupsInPandasWithStateSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateSuite] + enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite] + enableSuite[GlutenFlatMapGroupsWithStateSuite] enableSuite[GlutenFlatMapGroupsWithStateWithInitialStateSuite] enableSuite[GlutenGroupStateSuite] enableSuite[GlutenLongOffsetSuite] enableSuite[GlutenMemorySourceStressSuite] - // TODO: 4.x enableSuite[GlutenMultiStatefulOperatorsSuite] // 2 failures + enableSuite[GlutenMultiStatefulOperatorsSuite] enableSuite[GlutenReportSinkMetricsSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite] - // TODO: 4.x enableSuite[GlutenStreamSuite] - // TODO: 4.x enableSuite[GlutenStreamingAggregationDistributionSuite] - // TODO: 4.x enableSuite[GlutenStreamingAggregationSuite] - // TODO: 4.x enableSuite[GlutenStreamingDeduplicationDistributionSuite] - // TODO: 4.x enableSuite[GlutenStreamingDeduplicationSuite] + enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite] + enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite] + // Spark 4.x: these cases can hang waiting for expected failure with stateSchemaCheck off. + .excludeByPrefix("changing schema of state when restarting query - schema check off") + enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite] + enableSuite[GlutenStreamSuite] + enableSuite[GlutenStreamingAggregationDistributionSuite] + enableSuite[GlutenStreamingAggregationSuite] + // Spark 4.x: these cases can hang waiting for expected failure with stateSchemaCheck off. + .excludeByPrefix("changing schema of state when restarting query - schema check off") + enableSuite[GlutenStreamingDeduplicationDistributionSuite] + enableSuite[GlutenStreamingDeduplicationSuite] enableSuite[GlutenStreamingDeduplicationWithinWatermarkSuite] enableSuite[GlutenStreamingFullOuterJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingInnerJoinSuite] + enableSuite[GlutenStreamingInnerJoinSuite] enableSuite[GlutenStreamingLeftSemiJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingOuterJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingQueryHashPartitionVerifySuite] + enableSuite[GlutenStreamingOuterJoinSuite] + enableSuite[GlutenStreamingQueryHashPartitionVerifySuite] enableSuite[GlutenStreamingQueryListenerSuite] enableSuite[GlutenStreamingQueryListenersConfSuite] enableSuite[GlutenStreamingQueryManagerSuite] enableSuite[GlutenStreamingQueryOptimizationCorrectnessSuite] enableSuite[GlutenStreamingQueryStatusAndProgressSuite] enableSuite[GlutenStreamingSelfUnionSuite] - // TODO: 4.x enableSuite[GlutenStreamingSessionWindowDistributionSuite] + enableSuite[GlutenStreamingSessionWindowDistributionSuite] enableSuite[GlutenStreamingSessionWindowSuite] - // TODO: 4.x enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite] + enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite] enableSuite[GlutenStreamingSymmetricHashJoinHelperSuite] enableSuite[GlutenTransformWithListStateSuite] enableSuite[GlutenTransformWithListStateTTLSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala new file mode 100644 index 000000000000..210c1246057e --- /dev/null +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.SparkConf + +import java.net.JarURLConnection +import java.nio.file.{Files, Paths, StandardCopyOption} +import java.util.jar.JarFile + +private[sql] object GlutenStructuredStreamingResourceBootstrap { + private val StructuredStreamingRoot = "structured-streaming/" + private val ResourceProbe = s"${StructuredStreamingRoot}partition-tests/randomSchemas" + + @volatile private var initialized = false + + def ensureResourcesOnFilesystem(): Unit = synchronized { + if (initialized) { + return + } + + val maybeProbe = Option(getClass.getClassLoader.getResource(ResourceProbe)) + maybeProbe.foreach { + probeUrl => + if (probeUrl.getProtocol == "jar") { + copyStructuredStreamingResourcesFromJar(probeUrl) + } + } + + initialized = true + } + + private def copyStructuredStreamingResourcesFromJar(resourceUrl: java.net.URL): Unit = { + val maybeTestClassesRoot = Option(getClass.getResource("/")) + .filter(_.getProtocol == "file") + .map(url => Paths.get(url.toURI)) + if (maybeTestClassesRoot.isEmpty) { + return + } + + val testClassesRoot = maybeTestClassesRoot.get + if (Files.exists(testClassesRoot.resolve(ResourceProbe))) { + return + } + + val connection = resourceUrl.openConnection().asInstanceOf[JarURLConnection] + val jarPath = Paths.get(connection.getJarFileURL.toURI) + val jarFile = new JarFile(jarPath.toFile) + try { + val entries = jarFile.entries() + while (entries.hasMoreElements) { + val entry = entries.nextElement() + val entryName = entry.getName + if (!entry.isDirectory && entryName.startsWith(StructuredStreamingRoot)) { + val targetPath = testClassesRoot.resolve(entryName) + Option(targetPath.getParent).foreach(Files.createDirectories(_)) + val in = jarFile.getInputStream(entry) + try { + Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING) + } finally { + in.close() + } + } + } + } finally { + jarFile.close() + } + } +} + +private[sql] object GlutenStreamingTestConf { + def withFallbackToVanilla(conf: SparkConf): SparkConf = { + GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem() + conf + .set("spark.driver.host", "127.0.0.1") + .set("spark.driver.bindAddress", "127.0.0.1") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.shuffle.manager", "sort") + .set("spark.gluten.sql.columnar.batchscan", "false") + .set("spark.gluten.sql.columnar.filescan", "false") + .set("spark.gluten.sql.columnar.project", "false") + .set("spark.gluten.sql.columnar.filter", "false") + .set("spark.gluten.sql.columnar.sort", "false") + .set("spark.gluten.sql.columnar.window", "false") + .set("spark.gluten.sql.columnar.union", "false") + .set("spark.gluten.sql.columnar.expand", "false") + .set("spark.gluten.sql.columnar.generate", "false") + .set("spark.gluten.sql.columnar.coalesce", "false") + .set("spark.gluten.sql.columnar.range", "false") + .set("spark.gluten.sql.columnar.shuffle", "false") + .set("spark.gluten.sql.columnar.hashagg", "false") + .set("spark.gluten.sql.columnar.shuffledHashJoin", "false") + .set("spark.gluten.sql.columnar.sortMergeJoin", "false") + .set("spark.gluten.sql.columnar.broadcastExchange", "false") + .set("spark.gluten.sql.columnar.broadcastJoin", "false") + .set("spark.gluten.sql.columnar.appendData", "false") + .set("spark.gluten.sql.columnar.writeToDataSourceV2", "false") + .set("spark.gluten.sql.native.writer.enabled", "false") + .set("spark.gluten.sql.columnar.query.fallback.threshold", "0") + .set("spark.gluten.sql.columnar.wholeStage.fallback.threshold", "0") + .set("spark.gluten.sql.columnar.fallback.expressions.threshold", "0") + .set("spark.gluten.sql.columnar.fallback.preferColumnar", "false") + .set("spark.gluten.expression.blacklist", "collect_list,collect_set") + } +} + +trait GlutenStreamingSQLTestsTrait extends GlutenSQLTestsTrait { + private val structuredStreamingResourcesInitialized: Unit = + GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem() + + override def sparkConf: SparkConf = { + GlutenStreamingTestConf.withFallbackToVanilla(super.sparkConf) + } +} + +trait GlutenStreamingVanillaFallbackTestsTrait extends GlutenStreamingSQLTestsTrait { + override def sparkConf: SparkConf = { + super.sparkConf + } +} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala index 2bf23fbac156..f4f97cea5bfe 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenAcceptsLatestSeenOffsetSuite extends AcceptsLatestSeenOffsetSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala index ae006ee38a19..e2699e97ecbf 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenEventTimeWatermarkSuite extends EventTimeWatermarkSuite with GlutenSQLTestsTrait {} +class GlutenEventTimeWatermarkSuite + extends EventTimeWatermarkSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala index 74091348ea57..9d4ae3f7ff4e 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with GlutenSQLTestsTrait {} +class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with GlutenStreamingSQLTestsTrait {} -class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with GlutenSQLTestsTrait {} +class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala index fd681ffd1047..b953f5a4ffe9 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala @@ -16,10 +16,10 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with GlutenSQLTestsTrait {} +class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with GlutenStreamingSQLTestsTrait {} class GlutenFileStreamSourceStressTestSuite extends FileStreamSourceStressTestSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala index 566a5defe17c..2df2b99101d5 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamStressSuite extends FileStreamStressSuite with GlutenSQLTestsTrait {} +class GlutenFileStreamStressSuite extends FileStreamStressSuite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala index 2199e32a4029..5b2ec045909c 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsInPandasWithStateDistributionSuite extends FlatMapGroupsInPandasWithStateDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala index c653aa1d56d3..6d1ae5df2345 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsInPandasWithStateSuite extends FlatMapGroupsInPandasWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala index 343832f7ae4c..e9a0eee66c75 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenFlatMapGroupsWithStateDistributionSuite extends FlatMapGroupsWithStateDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala index 21d890a6ccd2..df7f30c14fca 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala @@ -16,12 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsWithStateSuite extends FlatMapGroupsWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} class GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite extends RocksDBStateStoreFlatMapGroupsWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala index 8ad63059924e..f39b66480a1d 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsWithStateWithInitialStateSuite extends FlatMapGroupsWithStateWithInitialStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala index ed28286fa676..468093cf03c4 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenMemorySourceStressSuite extends MemorySourceStressSuite with GlutenSQLTestsTrait {} +class GlutenMemorySourceStressSuite + extends MemorySourceStressSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala index 5860bf3b580f..2177f48acbcb 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenMultiStatefulOperatorsSuite extends MultiStatefulOperatorsSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala index 32a340d7f9c7..052a1f6aff37 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenReportSinkMetricsSuite extends ReportSinkMetricsSuite with GlutenSQLTestsTrait {} +class GlutenReportSinkMetricsSuite + extends ReportSinkMetricsSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala index 72918810129c..bd162bd6db79 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenStreamSuite extends StreamSuite with GlutenSQLTestsTrait {} +class GlutenStreamSuite extends StreamSuite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala index 37928f16ecea..def343896f1b 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingAggregationDistributionSuite extends StreamingAggregationDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala index d7d2af6e0732..ab831aff95bd 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala @@ -16,10 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait -class GlutenStreamingAggregationSuite extends StreamingAggregationSuite with GlutenSQLTestsTrait {} +class GlutenStreamingAggregationSuite + extends StreamingAggregationSuite + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenRocksDBStateStoreStreamingAggregationSuite extends RocksDBStateStoreStreamingAggregationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala index b8adfbc86a46..8fcb8faeab64 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingDeduplicationDistributionSuite extends StreamingDeduplicationDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala index 67c28131418b..bfc50fc5ea45 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala @@ -16,12 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingDeduplicationSuite extends StreamingDeduplicationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} class GlutenRocksDBStateStoreStreamingDeduplicationSuite extends RocksDBStateStoreStreamingDeduplicationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala index dd555fd72a9a..e6aec892461e 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingDeduplicationWithinWatermarkSuite extends StreamingDeduplicationWithinWatermarkSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala index 65489c981df8..321c0fb559c5 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala @@ -16,16 +16,20 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait -class GlutenStreamingInnerJoinSuite extends StreamingInnerJoinSuite with GlutenSQLTestsTrait {} +class GlutenStreamingInnerJoinSuite + extends StreamingInnerJoinSuite + with GlutenStreamingVanillaFallbackTestsTrait {} -class GlutenStreamingOuterJoinSuite extends StreamingOuterJoinSuite with GlutenSQLTestsTrait {} +class GlutenStreamingOuterJoinSuite + extends StreamingOuterJoinSuite + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenStreamingFullOuterJoinSuite extends StreamingFullOuterJoinSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenStreamingLeftSemiJoinSuite extends StreamingLeftSemiJoinSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala index 82dbf96c7897..ec6b482533f1 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala @@ -16,8 +16,109 @@ */ package org.apache.spark.sql.streaming +import org.apache.spark.SparkConf +import org.apache.spark.sql.GlutenSQLTestsBaseTrait +import org.apache.spark.sql.GlutenStreamingTestConf import org.apache.spark.sql.GlutenTestsCommonTrait +import java.io.{File, FileOutputStream} +import java.nio.file.Files + class GlutenStreamingQueryHashPartitionVerifySuite - extends StreamingQueryHashPartitionVerifySuite - with GlutenTestsCommonTrait {} + extends { + private val sparkTestHomeInitialized: Unit = + GlutenStreamingQueryHashPartitionVerifySuite.initSparkTestHome() + } + with StreamingQueryHashPartitionVerifySuite + with GlutenTestsCommonTrait { + + override def sparkConf: SparkConf = { + val conf = super.sparkConf + val warehousePath = + conf + .getOption("spark.sql.warehouse.dir") + .getOrElse(System.getProperty("java.io.tmpdir") + "/spark-warehouse") + GlutenStreamingTestConf.withFallbackToVanilla( + GlutenSQLTestsBaseTrait.nativeSparkConf(conf, warehousePath)) + } +} + +object GlutenStreamingQueryHashPartitionVerifySuite { + private val ResourcePath = "structured-streaming/partition-tests" + private val PartitionTestsRelativePath = + "sql/core/src/test/resources/structured-streaming/partition-tests" + + private[streaming] def initSparkTestHome(): Unit = { + val existingValidSparkTestHome = + sys.props.get("spark.test.home").filter(isValidSparkTestHome) + if (existingValidSparkTestHome.isEmpty) { + val sparkTestHome = createSparkTestHomeFromResources() + .orElse(localSparkTestHome().filter(isValidSparkTestHome)) + .orElse(sys.env.get("SPARK_HOME").filter(isValidSparkTestHome)) + + sparkTestHome.foreach(path => System.setProperty("spark.test.home", path)) + } + } + + private def localSparkTestHome(): Option[String] = { + val userDir = sys.props.getOrElse("user.dir", ".") + val moduleRelativeSparkTestHome = + new File(userDir, "../common/src/test/resources/spark-home") + val localSparkTestHome = + new File(userDir, "src/test/resources/spark-home") + + if (moduleRelativeSparkTestHome.exists()) { + Some(moduleRelativeSparkTestHome.getAbsolutePath) + } else if (localSparkTestHome.exists()) { + Some(localSparkTestHome.getAbsolutePath) + } else { + None + } + } + + private def createSparkTestHomeFromResources(): Option[String] = { + val randomSchemasPath = s"$ResourcePath/randomSchemas" + val rowsAndPartIdsPath = s"$ResourcePath/rowsAndPartIds" + + val rootDir = Files.createTempDirectory("gluten-spark-test-home").toFile + val partitionTestsDir = new File(rootDir, PartitionTestsRelativePath) + + if (!partitionTestsDir.mkdirs() && !partitionTestsDir.exists()) { + None + } else { + val copiedRandomSchemas = + copyResource(randomSchemasPath, new File(partitionTestsDir, "randomSchemas")) + val copiedRowsAndPartIds = + copyResource(rowsAndPartIdsPath, new File(partitionTestsDir, "rowsAndPartIds")) + + if (copiedRandomSchemas && copiedRowsAndPartIds) { + rootDir.deleteOnExit() + Some(rootDir.getAbsolutePath).filter(isValidSparkTestHome) + } else { + None + } + } + } + + private def isValidSparkTestHome(path: String): Boolean = { + val partitionTestsDir = new File(path, PartitionTestsRelativePath) + val randomSchemas = new File(partitionTestsDir, "randomSchemas") + val rowsAndPartIds = new File(partitionTestsDir, "rowsAndPartIds") + randomSchemas.exists() && rowsAndPartIds.exists() + } + + private def copyResource(resourcePath: String, targetFile: File): Boolean = { + val in = Option(getClass.getClassLoader.getResourceAsStream(resourcePath)) + in.exists { + is => + val out = new FileOutputStream(targetFile) + try { + is.transferTo(out) + true + } finally { + out.close() + is.close() + } + } + } +} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala index 43793e599a6c..3f3122a64803 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryListenerSuite extends StreamingQueryListenerSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala index 785fa418ea88..1db79b542495 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryListenersConfSuite extends StreamingQueryListenersConfSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala index 5629f12becad..a93c88843551 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryManagerSuite extends StreamingQueryManagerSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala index 3503398148cc..a607af941f84 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryOptimizationCorrectnessSuite extends StreamingQueryOptimizationCorrectnessSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala index 921e20574fbf..bc3d6b20a2cf 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryStatusAndProgressSuite extends StreamingQueryStatusAndProgressSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala index 7f7000c33cf4..0926fc00a67d 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala @@ -16,12 +16,11 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec -import org.apache.spark.sql.execution.exchange.REQUIRED_BY_STATEFUL_OPERATOR +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait +import org.apache.spark.sql.execution.exchange.{REQUIRED_BY_STATEFUL_OPERATOR, ShuffleExchangeLike} import org.apache.spark.sql.execution.streaming.MemoryStream -class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenSQLTestsTrait { +class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenStreamingSQLTestsTrait { import testImplicits._ @@ -38,11 +37,13 @@ class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenSQLTestsT Execute { qe => val shuffleOpt = qe.lastExecution.executedPlan.collect { - case s: ColumnarShuffleExchangeExec => s + case s: ShuffleExchangeLike if s.shuffleOrigin == REQUIRED_BY_STATEFUL_OPERATOR => s } - assert(shuffleOpt.nonEmpty, "No shuffle exchange found in the query plan") - assert(shuffleOpt.head.shuffleOrigin === REQUIRED_BY_STATEFUL_OPERATOR) + assert( + shuffleOpt.nonEmpty, + s"No stateful shuffle exchange found in the query plan: " + + qe.lastExecution.executedPlan) } ) } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala index 4f5a84d6428a..fdab47efd754 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenStreamingSelfUnionSuite extends StreamingSelfUnionSuite with GlutenSQLTestsTrait {} +class GlutenStreamingSelfUnionSuite + extends StreamingSelfUnionSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala index de167d74f14f..d21a95ff01bf 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingSessionWindowDistributionSuite extends StreamingSessionWindowDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala index 00888b1a93bf..45e0c6eb2e63 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingSessionWindowSuite extends StreamingSessionWindowSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala index 2fa4b17fc914..acb7ebb3cef3 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingStateStoreFormatCompatibilitySuite extends StreamingStateStoreFormatCompatibilitySuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala index 5dd65c179fff..9020a2702b54 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingSymmetricHashJoinHelperSuite extends StreamingSymmetricHashJoinHelperSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala index 907e048a33dc..8ad2aafaff28 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithListStateSuite extends TransformWithListStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala index b785c7db3386..315021d65ad2 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithListStateTTLSuite extends TransformWithListStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala index b3742f6ee075..650a36271ebb 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithMapStateSuite extends TransformWithMapStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala index 48b4d634bd1c..0c20e0ce6227 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithMapStateTTLSuite extends TransformWithMapStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala index 61336607ed74..95aa8433b07c 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateAvroSuite extends TransformWithStateAvroSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala index 852f4161d89a..e2d475e997ac 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateChainingSuite extends TransformWithStateChainingSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala index 3ba4436b0a18..dbe1fefc5edb 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateClusterSuite extends TransformWithStateClusterSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala index 5727122b4aac..1ecfb105251a 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateInitialStateSuite extends TransformWithStateInitialStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala index 6c02a69c2c60..a0c6a2f99020 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateValidationSuite extends TransformWithStateValidationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala index 40802ce46c34..fde363a58087 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateUnsafeRowSuite extends TransformWithStateUnsafeRowSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala index 6aeb84f9f19e..1c5ec3190996 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithValueStateTTLSuite extends TransformWithValueStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala index 89fb3cf5f469..efab0a270b98 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenTriggerAvailableNowSuite extends TriggerAvailableNowSuite with GlutenSQLTestsTrait {} +class GlutenTriggerAvailableNowSuite + extends TriggerAvailableNowSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5baf6e186c73..616cb9ac7dac 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1171,8 +1171,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("detect escaped path and report the migration guide") .exclude("ignore the escaped path check when the flag is off") .excludeByPrefix("SPARK-51187") - // Rewrite for the query plan check - .excludeByPrefix("SPARK-49905") // TODO: fix on Spark-4.1 introduced by https://github.com/apache/spark/pull/52645 .exclude("SPARK-53942: changing the number of stateless shuffle partitions via config") .exclude("SPARK-53942: stateful shuffle partitions are retained from old checkpoint") @@ -1205,45 +1203,49 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSQLMetricsSuite] enableSuite[GlutenAcceptsLatestSeenOffsetSuite] enableSuite[GlutenCommitLogSuite] - // TODO: 4.x enableSuite[GlutenEventTimeWatermarkSuite] + enableSuite[GlutenEventTimeWatermarkSuite] enableSuite[GlutenFileStreamSinkV1Suite] - // TODO: 4.x enableSuite[GlutenFileStreamSinkV2Suite] // 1 failure + enableSuite[GlutenFileStreamSinkV2Suite] enableSuite[GlutenFileStreamSourceStressTestSuite] - // TODO: 4.x enableSuite[GlutenFileStreamSourceSuite] + enableSuite[GlutenFileStreamSourceSuite] enableSuite[GlutenFileStreamStressSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] // failures with GlutenPlugin + enableSuite[GlutenFlatMapGroupsInPandasWithStateDistributionSuite] enableSuite[GlutenFlatMapGroupsInPandasWithStateSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite] - // TODO: 4.x enableSuite[GlutenFlatMapGroupsWithStateSuite] + enableSuite[GlutenFlatMapGroupsWithStateDistributionSuite] + enableSuite[GlutenFlatMapGroupsWithStateSuite] enableSuite[GlutenFlatMapGroupsWithStateWithInitialStateSuite] enableSuite[GlutenGroupStateSuite] enableSuite[GlutenLongOffsetSuite] enableSuite[GlutenMemorySourceStressSuite] - // TODO: 4.x enableSuite[GlutenMultiStatefulOperatorsSuite] // 2 failures + enableSuite[GlutenMultiStatefulOperatorsSuite] enableSuite[GlutenReportSinkMetricsSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite] - // TODO: 4.x enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite] - // TODO: 4.x enableSuite[GlutenStreamSuite] - // TODO: 4.x enableSuite[GlutenStreamingAggregationDistributionSuite] - // TODO: 4.x enableSuite[GlutenStreamingAggregationSuite] - // TODO: 4.x enableSuite[GlutenStreamingDeduplicationDistributionSuite] - // TODO: 4.x enableSuite[GlutenStreamingDeduplicationSuite] + enableSuite[GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite] + enableSuite[GlutenRocksDBStateStoreStreamingAggregationSuite] + // Spark 4.x: these cases can hang waiting for expected failure with stateSchemaCheck off. + .excludeByPrefix("changing schema of state when restarting query - schema check off") + enableSuite[GlutenRocksDBStateStoreStreamingDeduplicationSuite] + enableSuite[GlutenStreamSuite] + enableSuite[GlutenStreamingAggregationDistributionSuite] + enableSuite[GlutenStreamingAggregationSuite] + // Spark 4.x: these cases can hang waiting for expected failure with stateSchemaCheck off. + .excludeByPrefix("changing schema of state when restarting query - schema check off") + enableSuite[GlutenStreamingDeduplicationDistributionSuite] + enableSuite[GlutenStreamingDeduplicationSuite] enableSuite[GlutenStreamingDeduplicationWithinWatermarkSuite] enableSuite[GlutenStreamingFullOuterJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingInnerJoinSuite] + enableSuite[GlutenStreamingInnerJoinSuite] enableSuite[GlutenStreamingLeftSemiJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingOuterJoinSuite] - // TODO: 4.x enableSuite[GlutenStreamingQueryHashPartitionVerifySuite] + enableSuite[GlutenStreamingOuterJoinSuite] + enableSuite[GlutenStreamingQueryHashPartitionVerifySuite] enableSuite[GlutenStreamingQueryListenerSuite] enableSuite[GlutenStreamingQueryListenersConfSuite] enableSuite[GlutenStreamingQueryManagerSuite] enableSuite[GlutenStreamingQueryOptimizationCorrectnessSuite] enableSuite[GlutenStreamingQueryStatusAndProgressSuite] enableSuite[GlutenStreamingSelfUnionSuite] - // TODO: 4.x enableSuite[GlutenStreamingSessionWindowDistributionSuite] + enableSuite[GlutenStreamingSessionWindowDistributionSuite] enableSuite[GlutenStreamingSessionWindowSuite] - // TODO: 4.x enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite] + enableSuite[GlutenStreamingStateStoreFormatCompatibilitySuite] enableSuite[GlutenStreamingSymmetricHashJoinHelperSuite] enableSuite[GlutenTransformWithListStateSuite] enableSuite[GlutenTransformWithListStateTTLSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala new file mode 100644 index 000000000000..210c1246057e --- /dev/null +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenStreamingTestsTraits.scala @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.SparkConf + +import java.net.JarURLConnection +import java.nio.file.{Files, Paths, StandardCopyOption} +import java.util.jar.JarFile + +private[sql] object GlutenStructuredStreamingResourceBootstrap { + private val StructuredStreamingRoot = "structured-streaming/" + private val ResourceProbe = s"${StructuredStreamingRoot}partition-tests/randomSchemas" + + @volatile private var initialized = false + + def ensureResourcesOnFilesystem(): Unit = synchronized { + if (initialized) { + return + } + + val maybeProbe = Option(getClass.getClassLoader.getResource(ResourceProbe)) + maybeProbe.foreach { + probeUrl => + if (probeUrl.getProtocol == "jar") { + copyStructuredStreamingResourcesFromJar(probeUrl) + } + } + + initialized = true + } + + private def copyStructuredStreamingResourcesFromJar(resourceUrl: java.net.URL): Unit = { + val maybeTestClassesRoot = Option(getClass.getResource("/")) + .filter(_.getProtocol == "file") + .map(url => Paths.get(url.toURI)) + if (maybeTestClassesRoot.isEmpty) { + return + } + + val testClassesRoot = maybeTestClassesRoot.get + if (Files.exists(testClassesRoot.resolve(ResourceProbe))) { + return + } + + val connection = resourceUrl.openConnection().asInstanceOf[JarURLConnection] + val jarPath = Paths.get(connection.getJarFileURL.toURI) + val jarFile = new JarFile(jarPath.toFile) + try { + val entries = jarFile.entries() + while (entries.hasMoreElements) { + val entry = entries.nextElement() + val entryName = entry.getName + if (!entry.isDirectory && entryName.startsWith(StructuredStreamingRoot)) { + val targetPath = testClassesRoot.resolve(entryName) + Option(targetPath.getParent).foreach(Files.createDirectories(_)) + val in = jarFile.getInputStream(entry) + try { + Files.copy(in, targetPath, StandardCopyOption.REPLACE_EXISTING) + } finally { + in.close() + } + } + } + } finally { + jarFile.close() + } + } +} + +private[sql] object GlutenStreamingTestConf { + def withFallbackToVanilla(conf: SparkConf): SparkConf = { + GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem() + conf + .set("spark.driver.host", "127.0.0.1") + .set("spark.driver.bindAddress", "127.0.0.1") + .set("spark.sql.shuffle.partitions", "5") + .set("spark.shuffle.manager", "sort") + .set("spark.gluten.sql.columnar.batchscan", "false") + .set("spark.gluten.sql.columnar.filescan", "false") + .set("spark.gluten.sql.columnar.project", "false") + .set("spark.gluten.sql.columnar.filter", "false") + .set("spark.gluten.sql.columnar.sort", "false") + .set("spark.gluten.sql.columnar.window", "false") + .set("spark.gluten.sql.columnar.union", "false") + .set("spark.gluten.sql.columnar.expand", "false") + .set("spark.gluten.sql.columnar.generate", "false") + .set("spark.gluten.sql.columnar.coalesce", "false") + .set("spark.gluten.sql.columnar.range", "false") + .set("spark.gluten.sql.columnar.shuffle", "false") + .set("spark.gluten.sql.columnar.hashagg", "false") + .set("spark.gluten.sql.columnar.shuffledHashJoin", "false") + .set("spark.gluten.sql.columnar.sortMergeJoin", "false") + .set("spark.gluten.sql.columnar.broadcastExchange", "false") + .set("spark.gluten.sql.columnar.broadcastJoin", "false") + .set("spark.gluten.sql.columnar.appendData", "false") + .set("spark.gluten.sql.columnar.writeToDataSourceV2", "false") + .set("spark.gluten.sql.native.writer.enabled", "false") + .set("spark.gluten.sql.columnar.query.fallback.threshold", "0") + .set("spark.gluten.sql.columnar.wholeStage.fallback.threshold", "0") + .set("spark.gluten.sql.columnar.fallback.expressions.threshold", "0") + .set("spark.gluten.sql.columnar.fallback.preferColumnar", "false") + .set("spark.gluten.expression.blacklist", "collect_list,collect_set") + } +} + +trait GlutenStreamingSQLTestsTrait extends GlutenSQLTestsTrait { + private val structuredStreamingResourcesInitialized: Unit = + GlutenStructuredStreamingResourceBootstrap.ensureResourcesOnFilesystem() + + override def sparkConf: SparkConf = { + GlutenStreamingTestConf.withFallbackToVanilla(super.sparkConf) + } +} + +trait GlutenStreamingVanillaFallbackTestsTrait extends GlutenStreamingSQLTestsTrait { + override def sparkConf: SparkConf = { + super.sparkConf + } +} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala index 2bf23fbac156..f4f97cea5bfe 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenAcceptsLatestSeenOffsetSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenAcceptsLatestSeenOffsetSuite extends AcceptsLatestSeenOffsetSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala index ae006ee38a19..e2699e97ecbf 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenEventTimeWatermarkSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenEventTimeWatermarkSuite extends EventTimeWatermarkSuite with GlutenSQLTestsTrait {} +class GlutenEventTimeWatermarkSuite + extends EventTimeWatermarkSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala index 74091348ea57..9d4ae3f7ff4e 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSinkSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with GlutenSQLTestsTrait {} +class GlutenFileStreamSinkV1Suite extends FileStreamSinkV1Suite with GlutenStreamingSQLTestsTrait {} -class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with GlutenSQLTestsTrait {} +class GlutenFileStreamSinkV2Suite extends FileStreamSinkV2Suite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala index fd681ffd1047..b953f5a4ffe9 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamSourceSuite.scala @@ -16,10 +16,10 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with GlutenSQLTestsTrait {} +class GlutenFileStreamSourceSuite extends FileStreamSourceSuite with GlutenStreamingSQLTestsTrait {} class GlutenFileStreamSourceStressTestSuite extends FileStreamSourceStressTestSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala index 566a5defe17c..2df2b99101d5 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFileStreamStressSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenFileStreamStressSuite extends FileStreamStressSuite with GlutenSQLTestsTrait {} +class GlutenFileStreamStressSuite extends FileStreamStressSuite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala index 2199e32a4029..5b2ec045909c 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsInPandasWithStateDistributionSuite extends FlatMapGroupsInPandasWithStateDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala index c653aa1d56d3..6d1ae5df2345 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsInPandasWithStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsInPandasWithStateSuite extends FlatMapGroupsInPandasWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala index 343832f7ae4c..e9a0eee66c75 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenFlatMapGroupsWithStateDistributionSuite extends FlatMapGroupsWithStateDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala index 21d890a6ccd2..df7f30c14fca 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateSuite.scala @@ -16,12 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsWithStateSuite extends FlatMapGroupsWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} class GlutenRocksDBStateStoreFlatMapGroupsWithStateSuite extends RocksDBStateStoreFlatMapGroupsWithStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala index 8ad63059924e..f39b66480a1d 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenFlatMapGroupsWithStateWithInitialStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenFlatMapGroupsWithStateWithInitialStateSuite extends FlatMapGroupsWithStateWithInitialStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala index ed28286fa676..468093cf03c4 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMemorySourceStressSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenMemorySourceStressSuite extends MemorySourceStressSuite with GlutenSQLTestsTrait {} +class GlutenMemorySourceStressSuite + extends MemorySourceStressSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala index 5860bf3b580f..2177f48acbcb 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenMultiStatefulOperatorsSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenMultiStatefulOperatorsSuite extends MultiStatefulOperatorsSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala index 32a340d7f9c7..052a1f6aff37 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenReportSinkMetricsSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenReportSinkMetricsSuite extends ReportSinkMetricsSuite with GlutenSQLTestsTrait {} +class GlutenReportSinkMetricsSuite + extends ReportSinkMetricsSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala index 2aaf0d8eea10..75268ae860d5 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeAllowlistSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamRealTimeModeAllowlistSuite extends StreamRealTimeModeAllowlistSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala index c9effaf52ab0..92919883e308 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeE2ESuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamRealTimeModeE2ESuite extends StreamRealTimeModeE2ESuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala index de80b65b642d..687d0e819a36 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamRealTimeModeSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenStreamRealTimeModeSuite extends StreamRealTimeModeSuite with GlutenSQLTestsTrait {} +class GlutenStreamRealTimeModeSuite + extends StreamRealTimeModeSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala index 72918810129c..bd162bd6db79 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenStreamSuite extends StreamSuite with GlutenSQLTestsTrait {} +class GlutenStreamSuite extends StreamSuite with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala index 37928f16ecea..def343896f1b 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingAggregationDistributionSuite extends StreamingAggregationDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala index d7d2af6e0732..ab831aff95bd 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingAggregationSuite.scala @@ -16,10 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait -class GlutenStreamingAggregationSuite extends StreamingAggregationSuite with GlutenSQLTestsTrait {} +class GlutenStreamingAggregationSuite + extends StreamingAggregationSuite + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenRocksDBStateStoreStreamingAggregationSuite extends RocksDBStateStoreStreamingAggregationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala index b8adfbc86a46..8fcb8faeab64 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingDeduplicationDistributionSuite extends StreamingDeduplicationDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala index 67c28131418b..bfc50fc5ea45 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationSuite.scala @@ -16,12 +16,12 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingDeduplicationSuite extends StreamingDeduplicationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} class GlutenRocksDBStateStoreStreamingDeduplicationSuite extends RocksDBStateStoreStreamingDeduplicationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala index dd555fd72a9a..e6aec892461e 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingDeduplicationWithinWatermarkSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingDeduplicationWithinWatermarkSuite extends StreamingDeduplicationWithinWatermarkSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala index 65489c981df8..321c0fb559c5 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingJoinSuite.scala @@ -16,16 +16,20 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait -class GlutenStreamingInnerJoinSuite extends StreamingInnerJoinSuite with GlutenSQLTestsTrait {} +class GlutenStreamingInnerJoinSuite + extends StreamingInnerJoinSuite + with GlutenStreamingVanillaFallbackTestsTrait {} -class GlutenStreamingOuterJoinSuite extends StreamingOuterJoinSuite with GlutenSQLTestsTrait {} +class GlutenStreamingOuterJoinSuite + extends StreamingOuterJoinSuite + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenStreamingFullOuterJoinSuite extends StreamingFullOuterJoinSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} class GlutenStreamingLeftSemiJoinSuite extends StreamingLeftSemiJoinSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala index 82dbf96c7897..ec6b482533f1 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryHashPartitionVerifySuite.scala @@ -16,8 +16,109 @@ */ package org.apache.spark.sql.streaming +import org.apache.spark.SparkConf +import org.apache.spark.sql.GlutenSQLTestsBaseTrait +import org.apache.spark.sql.GlutenStreamingTestConf import org.apache.spark.sql.GlutenTestsCommonTrait +import java.io.{File, FileOutputStream} +import java.nio.file.Files + class GlutenStreamingQueryHashPartitionVerifySuite - extends StreamingQueryHashPartitionVerifySuite - with GlutenTestsCommonTrait {} + extends { + private val sparkTestHomeInitialized: Unit = + GlutenStreamingQueryHashPartitionVerifySuite.initSparkTestHome() + } + with StreamingQueryHashPartitionVerifySuite + with GlutenTestsCommonTrait { + + override def sparkConf: SparkConf = { + val conf = super.sparkConf + val warehousePath = + conf + .getOption("spark.sql.warehouse.dir") + .getOrElse(System.getProperty("java.io.tmpdir") + "/spark-warehouse") + GlutenStreamingTestConf.withFallbackToVanilla( + GlutenSQLTestsBaseTrait.nativeSparkConf(conf, warehousePath)) + } +} + +object GlutenStreamingQueryHashPartitionVerifySuite { + private val ResourcePath = "structured-streaming/partition-tests" + private val PartitionTestsRelativePath = + "sql/core/src/test/resources/structured-streaming/partition-tests" + + private[streaming] def initSparkTestHome(): Unit = { + val existingValidSparkTestHome = + sys.props.get("spark.test.home").filter(isValidSparkTestHome) + if (existingValidSparkTestHome.isEmpty) { + val sparkTestHome = createSparkTestHomeFromResources() + .orElse(localSparkTestHome().filter(isValidSparkTestHome)) + .orElse(sys.env.get("SPARK_HOME").filter(isValidSparkTestHome)) + + sparkTestHome.foreach(path => System.setProperty("spark.test.home", path)) + } + } + + private def localSparkTestHome(): Option[String] = { + val userDir = sys.props.getOrElse("user.dir", ".") + val moduleRelativeSparkTestHome = + new File(userDir, "../common/src/test/resources/spark-home") + val localSparkTestHome = + new File(userDir, "src/test/resources/spark-home") + + if (moduleRelativeSparkTestHome.exists()) { + Some(moduleRelativeSparkTestHome.getAbsolutePath) + } else if (localSparkTestHome.exists()) { + Some(localSparkTestHome.getAbsolutePath) + } else { + None + } + } + + private def createSparkTestHomeFromResources(): Option[String] = { + val randomSchemasPath = s"$ResourcePath/randomSchemas" + val rowsAndPartIdsPath = s"$ResourcePath/rowsAndPartIds" + + val rootDir = Files.createTempDirectory("gluten-spark-test-home").toFile + val partitionTestsDir = new File(rootDir, PartitionTestsRelativePath) + + if (!partitionTestsDir.mkdirs() && !partitionTestsDir.exists()) { + None + } else { + val copiedRandomSchemas = + copyResource(randomSchemasPath, new File(partitionTestsDir, "randomSchemas")) + val copiedRowsAndPartIds = + copyResource(rowsAndPartIdsPath, new File(partitionTestsDir, "rowsAndPartIds")) + + if (copiedRandomSchemas && copiedRowsAndPartIds) { + rootDir.deleteOnExit() + Some(rootDir.getAbsolutePath).filter(isValidSparkTestHome) + } else { + None + } + } + } + + private def isValidSparkTestHome(path: String): Boolean = { + val partitionTestsDir = new File(path, PartitionTestsRelativePath) + val randomSchemas = new File(partitionTestsDir, "randomSchemas") + val rowsAndPartIds = new File(partitionTestsDir, "rowsAndPartIds") + randomSchemas.exists() && rowsAndPartIds.exists() + } + + private def copyResource(resourcePath: String, targetFile: File): Boolean = { + val in = Option(getClass.getClassLoader.getResourceAsStream(resourcePath)) + in.exists { + is => + val out = new FileOutputStream(targetFile) + try { + is.transferTo(out) + true + } finally { + out.close() + is.close() + } + } + } +} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala index 43793e599a6c..3f3122a64803 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenerSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryListenerSuite extends StreamingQueryListenerSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala index 785fa418ea88..1db79b542495 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryListenersConfSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryListenersConfSuite extends StreamingQueryListenersConfSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala index 5629f12becad..a93c88843551 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryManagerSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryManagerSuite extends StreamingQueryManagerSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala index 3503398148cc..a607af941f84 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryOptimizationCorrectnessSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryOptimizationCorrectnessSuite extends StreamingQueryOptimizationCorrectnessSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala index 921e20574fbf..bc3d6b20a2cf 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQueryStatusAndProgressSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingQueryStatusAndProgressSuite extends StreamingQueryStatusAndProgressSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala index 87e7924ce790..bf9c890c7842 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingQuerySuite.scala @@ -16,12 +16,11 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait -import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec -import org.apache.spark.sql.execution.exchange.REQUIRED_BY_STATEFUL_OPERATOR +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait +import org.apache.spark.sql.execution.exchange.{REQUIRED_BY_STATEFUL_OPERATOR, ShuffleExchangeLike} import org.apache.spark.sql.execution.streaming.runtime.MemoryStream -class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenSQLTestsTrait { +class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenStreamingSQLTestsTrait { import testImplicits._ @@ -38,11 +37,13 @@ class GlutenStreamingQuerySuite extends StreamingQuerySuite with GlutenSQLTestsT Execute { qe => val shuffleOpt = qe.lastExecution.executedPlan.collect { - case s: ColumnarShuffleExchangeExec => s + case s: ShuffleExchangeLike if s.shuffleOrigin == REQUIRED_BY_STATEFUL_OPERATOR => s } - assert(shuffleOpt.nonEmpty, "No shuffle exchange found in the query plan") - assert(shuffleOpt.head.shuffleOrigin === REQUIRED_BY_STATEFUL_OPERATOR) + assert( + shuffleOpt.nonEmpty, + s"No stateful shuffle exchange found in the query plan: " + + qe.lastExecution.executedPlan) } ) } diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala index 4f5a84d6428a..fdab47efd754 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSelfUnionSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenStreamingSelfUnionSuite extends StreamingSelfUnionSuite with GlutenSQLTestsTrait {} +class GlutenStreamingSelfUnionSuite + extends StreamingSelfUnionSuite + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala index de167d74f14f..d21a95ff01bf 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowDistributionSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingSessionWindowDistributionSuite extends StreamingSessionWindowDistributionSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala index 00888b1a93bf..45e0c6eb2e63 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSessionWindowSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingSessionWindowSuite extends StreamingSessionWindowSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala index 2fa4b17fc914..acb7ebb3cef3 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingStateStoreFormatCompatibilitySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingVanillaFallbackTestsTrait class GlutenStreamingStateStoreFormatCompatibilitySuite extends StreamingStateStoreFormatCompatibilitySuite - with GlutenSQLTestsTrait {} + with GlutenStreamingVanillaFallbackTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala index 5dd65c179fff..9020a2702b54 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenStreamingSymmetricHashJoinHelperSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenStreamingSymmetricHashJoinHelperSuite extends StreamingSymmetricHashJoinHelperSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala index 907e048a33dc..8ad2aafaff28 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithListStateSuite extends TransformWithListStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala index b785c7db3386..315021d65ad2 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithListStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithListStateTTLSuite extends TransformWithListStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala index b3742f6ee075..650a36271ebb 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithMapStateSuite extends TransformWithMapStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala index 48b4d634bd1c..0c20e0ce6227 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithMapStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithMapStateTTLSuite extends TransformWithMapStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala index 61336607ed74..95aa8433b07c 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateAvroSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateAvroSuite extends TransformWithStateAvroSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala index 852f4161d89a..e2d475e997ac 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateChainingSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateChainingSuite extends TransformWithStateChainingSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala index 3ba4436b0a18..dbe1fefc5edb 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateClusterSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateClusterSuite extends TransformWithStateClusterSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala index 5727122b4aac..1ecfb105251a 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateInitialStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateInitialStateSuite extends TransformWithStateInitialStateSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala index 6c02a69c2c60..a0c6a2f99020 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateValidationSuite extends TransformWithStateValidationSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala index 40802ce46c34..fde363a58087 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithStateUnsafeRowSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithStateUnsafeRowSuite extends TransformWithStateUnsafeRowSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala index 6aeb84f9f19e..1c5ec3190996 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTransformWithValueStateTTLSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait class GlutenTransformWithValueStateTTLSuite extends TransformWithValueStateTTLSuite - with GlutenSQLTestsTrait {} + with GlutenStreamingSQLTestsTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala index 89fb3cf5f469..efab0a270b98 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/streaming/GlutenTriggerAvailableNowSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.streaming -import org.apache.spark.sql.GlutenSQLTestsTrait +import org.apache.spark.sql.GlutenStreamingSQLTestsTrait -class GlutenTriggerAvailableNowSuite extends TriggerAvailableNowSuite with GlutenSQLTestsTrait {} +class GlutenTriggerAvailableNowSuite + extends TriggerAvailableNowSuite + with GlutenStreamingSQLTestsTrait {}