Skip to content

Drill-4309: Make this option store.hive.optimize_scan_with_native_readers=true defaul #625

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vdiravka wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Drill-4309: Make this option store.hive.optimize_scan_with_native_readers=true defaul #625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5fe7d5c
DRILL-4309: Make this option `store.hive.optimize_scan_with_native_re…
hsuanyi Jan 25, 2016
34a87a3
DRILL-4309: Make this option `store.hive.optimize_scan_with_native_r…
vdiravka Oct 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 7 additions & 12 deletions contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHivePartitionPruning.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,21 +118,16 @@ public void pruneDataTypeSupport() throws Exception {

@Test
public void pruneDataTypeSupportNativeReaders() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
final String query = "EXPLAIN PLAN FOR " +
"SELECT * FROM hive.readtest_parquet WHERE tinyint_part = 64";
final String query = "EXPLAIN PLAN FOR " +
"SELECT * FROM hive.readtest_parquet WHERE tinyint_part = 64";

final String plan = getPlanInString(query, OPTIQ_FORMAT);
final String plan = getPlanInString(query, OPTIQ_FORMAT);

// Check and make sure that Filter is not present in the plan
assertFalse(plan.contains("Filter"));
// Check and make sure that Filter is not present in the plan
assertFalse(plan.contains("Filter"));

// Make sure the plan contains the Hive scan utilizing native parquet reader
assertTrue(plan.contains("groupscan=[HiveDrillNativeParquetScan"));
} finally {
test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
}
// Make sure the plan contains the Hive scan utilizing native parquet reader
assertTrue(plan.contains("groupscan=[HiveDrillNativeParquetScan"));
}

@Test // DRILL-3579
Expand Down
13 changes: 4 additions & 9 deletions contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,9 @@ public void testHiveCountStar() throws Exception {

@Test
public void projectPushDownOnHiveParquetTable() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
String query = "SELECT boolean_field, boolean_part, int_field, int_part FROM hive.readtest_parquet";
String expectedColNames = "\"columns\" : [ \"`boolean_field`\", \"`dir0`\", \"`int_field`\", \"`dir9`\" ]";

testHelper(query, 2, expectedColNames, "hive-drill-native-parquet-scan");
} finally {
test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
}
String query = "SELECT boolean_field, boolean_part, int_field, int_part FROM hive.readtest_parquet";
String expectedColNames = "\"columns\" : [ \"`boolean_field`\", \"`dir0`\", \"`int_field`\", \"`dir9`\" ]";

testHelper(query, 2, expectedColNames, "hive-drill-native-parquet-scan");
}
}
273 changes: 125 additions & 148 deletions contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
Expand All @@ -19,7 +19,6 @@

import com.google.common.collect.ImmutableMap;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.hadoop.fs.FileSystem;
import org.joda.time.DateTime;
Expand All @@ -45,23 +44,15 @@ public static void setupOptions() throws Exception {

@Test // DRILL-4083
public void testNativeScanWhenNoColumnIsRead() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
String query = "SELECT count(*) as col FROM hive.countStar_Parquet";
testPhysicalPlan(query, "hive-drill-native-parquet-scan");

String query = "SELECT count(*) as col FROM hive.countStar_Parquet";
testPhysicalPlan(query, "hive-drill-native-parquet-scan");

testBuilder()
.sqlQuery(query)
.unOrdered()
.baselineColumns("col")
.baselineValues(200l)
.go();
} finally {
test(String.format("alter session set `%s` = %s",
ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS,
ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR.getDefault().bool_val ? "true" : "false"));
}
testBuilder()
.sqlQuery(query)
.unOrdered()
.baselineColumns("col")
.baselineValues(200l)
.go();
}

@Test
Expand Down Expand Up @@ -204,114 +195,111 @@ public void readAllSupportedHiveDataTypes() throws Exception {
*/
@Test
public void readAllSupportedHiveDataTypesNativeParquet() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
final String query = "SELECT * FROM hive.readtest_parquet";

// Make sure the plan has Hive scan with native parquet reader
testPhysicalPlan(query, "hive-drill-native-parquet-scan");

testBuilder().sqlQuery(query)
.unOrdered()
.baselineColumns(
"binary_field",
"boolean_field",
"tinyint_field",
"decimal0_field",
"decimal9_field",
"decimal18_field",
"decimal28_field",
"decimal38_field",
"double_field",
"float_field",
"int_field",
"bigint_field",
"smallint_field",
"string_field",
"varchar_field",
"timestamp_field",
"char_field",
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary_part",
"boolean_part",
"tinyint_part",
"decimal0_part",
"decimal9_part",
"decimal18_part",
"decimal28_part",
"decimal38_part",
"double_part",
"float_part",
"int_part",
"bigint_part",
"smallint_part",
"string_part",
"varchar_part",
"timestamp_part",
"date_part",
"char_part")
.baselineValues(
"binaryfield".getBytes(),
false,
34,
new BigDecimal("66"),
new BigDecimal("2347.92"),
new BigDecimal("2758725827.99990"),
new BigDecimal("29375892739852.8"),
new BigDecimal("89853749534593985.783"),
8.345d,
4.67f,
123456,
234235L,
3455,
"stringfield",
"varcharfield",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
"charfield",
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary",
true,
64,
new BigDecimal("37"),
new BigDecimal("36.90"),
new BigDecimal("3289379872.94565"),
new BigDecimal("39579334534534.4"),
new BigDecimal("363945093845093890.900"),
8.345d,
4.67f,
123456,
234235L,
3455,
"string",
"varchar",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
new DateTime(Date.valueOf("2013-07-05").getTime()),
"char")
.baselineValues( // All fields are null, but partition fields have non-null values
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary",
true,
64,
new BigDecimal("37"),
new BigDecimal("36.90"),
new BigDecimal("3289379872.94565"),
new BigDecimal("39579334534534.4"),
new BigDecimal("363945093845093890.900"),
8.345d,
4.67f,
123456,
234235L,
3455,
"string",
"varchar",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
new DateTime(Date.valueOf("2013-07-05").getTime()),
"char")
.go();
} finally {
test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
}
final String query = "SELECT * FROM hive.readtest_parquet";

// Make sure the plan has Hive scan with native parquet reader
testPhysicalPlan(query, "hive-drill-native-parquet-scan");

testBuilder().sqlQuery(query)
.unOrdered()
.baselineColumns(
"binary_field",
"boolean_field",
"tinyint_field",
"decimal0_field",
"decimal9_field",
"decimal18_field",
"decimal28_field",
"decimal38_field",
"double_field",
"float_field",
"int_field",
"bigint_field",
"smallint_field",
"string_field",
"varchar_field",
"timestamp_field",
"date_field",
"char_field",
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary_part",
"boolean_part",
"tinyint_part",
"decimal0_part",
"decimal9_part",
"decimal18_part",
"decimal28_part",
"decimal38_part",
"double_part",
"float_part",
"int_part",
"bigint_part",
"smallint_part",
"string_part",
"varchar_part",
"timestamp_part",
"date_part",
"char_part")
.baselineValues(
"binaryfield".getBytes(),
false,
34,
new BigDecimal("66"),
new BigDecimal("2347.92"),
new BigDecimal("2758725827.99990"),
new BigDecimal("29375892739852.8"),
new BigDecimal("89853749534593985.783"),
8.345d,
4.67f,
123456,
234235L,
3455,
"stringfield",
"varcharfield",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
new DateTime(Date.valueOf("2013-07-05").getTime()),
"charfield",
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary",
true,
64,
new BigDecimal("37"),
new BigDecimal("36.90"),
new BigDecimal("3289379872.94565"),
new BigDecimal("39579334534534.4"),
new BigDecimal("363945093845093890.900"),
8.345d,
4.67f,
123456,
234235L,
3455,
"string",
"varchar",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
new DateTime(Date.valueOf("2013-07-05").getTime()),
"char")
.baselineValues( // All fields are null, but partition fields have non-null values
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
// There is a regression in Hive 1.2.1 in binary and boolean partition columns. Disable for now.
//"binary",
true,
64,
new BigDecimal("37"),
new BigDecimal("36.90"),
new BigDecimal("3289379872.94565"),
new BigDecimal("39579334534534.4"),
new BigDecimal("363945093845093890.900"),
8.345d,
4.67f,
123456,
234235L,
3455,
"string",
"varchar",
new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
new DateTime(Date.valueOf("2013-07-05").getTime()),
"char")
.go();
}

@Test
Expand Down Expand Up @@ -385,16 +373,11 @@ public void readFromAlteredPartitionedTable() throws Exception {

@Test // DRILL-3938
public void nativeReaderIsDisabledForAlteredPartitionedTable() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
final String query = "EXPLAIN PLAN FOR SELECT key, `value`, newcol FROM hive.kv_parquet ORDER BY key LIMIT 1";

// Make sure the HiveScan in plan has no native parquet reader
final String planStr = getPlanInString(query, JSON_FORMAT);
assertFalse("Hive native is not expected in the plan", planStr.contains("hive-drill-native-parquet-scan"));
} finally {
test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
}
final String query = "EXPLAIN PLAN FOR SELECT key, `value`, newcol FROM hive.kv_parquet ORDER BY key LIMIT 1";

// Make sure the HiveScan in plan has no native parquet reader
final String planStr = getPlanInString(query, JSON_FORMAT);
assertFalse("Hive native is not expected in the plan", planStr.contains("hive-drill-native-parquet-scan"));
}

@Test // DRILL-3739
Expand All @@ -409,18 +392,12 @@ public void readingFromStorageHandleBasedTable() throws Exception {

@Test // DRILL-3739
public void readingFromStorageHandleBasedTable2() throws Exception {
try {
test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));

testBuilder()
.sqlQuery("SELECT * FROM hive.kv_sh ORDER BY key LIMIT 2")
.ordered()
.baselineColumns("key", "value")
.expectsEmptyResultSet()
.go();
} finally {
test(String.format("alter session set `%s` = false", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
}
testBuilder()
.sqlQuery("SELECT * FROM hive.kv_sh ORDER BY key LIMIT 2")
.ordered()
.baselineColumns("key", "value")
.expectsEmptyResultSet()
.go();
}

@Test // DRILL-3688
Expand Down
4 changes: 3 additions & 1 deletion ...orage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
Expand Down Expand Up @@ -330,6 +330,7 @@ private void generateTestData() throws Exception {
" string_field STRING," +
" varchar_field VARCHAR(50)," +
" timestamp_field TIMESTAMP," +
" date_field DATE," +
" char_field CHAR(10)" +
") PARTITIONED BY (" +
// There is a regression in Hive 1.2.1 in binary type partition columns. Disable for now.
Expand Down Expand Up @@ -393,6 +394,7 @@ private void generateTestData() throws Exception {
" string_field," +
" varchar_field," +
" timestamp_field," +
" date_field," +
" char_field" +
" FROM readtest WHERE tinyint_part = 64");

Expand Down
2 changes: 1 addition & 1 deletion exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ public interface ExecConstants {
// in core which is not right. Move this option and above two mongo plugin related options once we have the feature.
String HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS = "store.hive.optimize_scan_with_native_readers";
OptionValidator HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR =
new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, false);
new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, true);

String SLICE_TARGET = "planner.slice_target";
long SLICE_TARGET_DEFAULT = 100000l;
Expand Down