Feature/missing hana tests

backend/pom.xml

@@ -373,6 +373,11 @@
             <artifactId>testcontainers-postgresql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers-clickhouse</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers-solr</artifactId>
@@ -383,6 +388,12 @@
             <artifactId>ngdbc</artifactId>
             <version>2.17.10</version>
         </dependency>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.9.8</version>
+            <classifier>all</classifier>
+        </dependency>
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_dropwizard</artifactId>

backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java

@@ -33,6 +33,7 @@
 import org.jooq.Select;
 import org.jooq.SelectConditionStep;
 import org.jooq.Table;
+import org.jooq.impl.DSL;
 
 @RequiredArgsConstructor
 public class SqlEntityResolver implements EntityResolver {
@@ -144,7 +145,8 @@ private Map<Integer, IdResolveInfo> resolveIds(String[][] values, List<Function<
 		Field<Integer> rowIndex = field(name(ROW_INDEX), Integer.class);
 		Field<String> externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class);
 		Field<String> innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class);
-		Field<Boolean> isResolved = innerPrimaryColumn.isNotNull().as(IS_RESOLVED_ALIAS);
+		// Would prefer this to be `is not null`, but hana does not support that for fields
+		Field<Boolean> isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS);
 
 		Table<Record> allIdsTable = table(name(idColumns.getTable()));
 

backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java

@@ -1,8 +1,9 @@
 package com.bakdata.conquery.models.config;
 
 import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle;
-import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle;
+import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 
@@ -15,13 +16,8 @@
 @Getter
 public enum Dialect {
 
-	/**
-	 * Dialect for PostgreSQL database
-	 */
 	POSTGRESQL(new PostgreDialectBundle()),
-	/**
-	 * Dialect for SAP HANA database
-	 */
+	CLICKHOUSE(new ClickhouseDialectBundle()),
 	HANA(new HanaDialectBundle());
 
 	private final DialectBundle dialectBundle;

backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java

@@ -78,6 +78,10 @@ public Collection<ResultAsset> generateResultURLs(ManagedExecution exec, UriBuil
 			return Collections.emptyList();
 		}
 
+		if(singleExecution.getResultInfos() == null){
+			return Collections.emptyList();
+		}
+
 		// Save id column count to later check if xlsx dimensions are feasible
 		idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size();
 

backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java

@@ -14,6 +14,7 @@
 import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeCache;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector;
@@ -95,7 +96,7 @@ public static CBlock createCBlock(ConceptTreeConnector connector, BucketId bucke
 		final int[][] mostSpecificChildren = calculateSpecificChildrenPaths(bucket, connector, bucketManager);
 		//TODO Object2LongMap
 		final Map<String, Long> includedConcepts = calculateConceptElementPathBloomFilter(bucket, mostSpecificChildren);
-		final Map<String, CDateRange> entitySpans = calculateEntityDateIndices(bucket);
+		final Map<String, CDateRange> entitySpans = calculateEntityDateIndices(bucket, connector);
 
 		final CBlock cBlock = new CBlock(bucketId, connector.getId(), includedConcepts, entitySpans, mostSpecificChildren);
 		return cBlock;
@@ -147,17 +148,10 @@ private static Map<String, Long> calculateConceptElementPathBloomFilter(Bucket b
 	 *
 	 * @implNote This is an unrolled implementation of {@link CDateRange#span(CDateRange)}.
 	 */
-	private static Map<String, CDateRange> calculateEntityDateIndices(Bucket bucket) {
+	private static Map<String, CDateRange> calculateEntityDateIndices(Bucket bucket, ConceptTreeConnector connector) {
 		final Map<String, CDateRange> spans = new HashMap<>();
 
-		final Table table = bucket.getTable().resolve();
-
-
-		for (Column column : table.getColumns()) {
-			if (!column.getType().isDateCompatible()) {
-				continue;
-			}
-
+		for (ValidityDate validityDate : connector.getValidityDates()) {
 			for (String entity : bucket.entities()) {
 				final int end = bucket.getEntityEnd(entity);
 
@@ -166,14 +160,14 @@ private static Map<String, CDateRange> calculateEntityDateIndices(Bucket bucket)
 				int max = Integer.MIN_VALUE;
 				int min = Integer.MAX_VALUE;
 
-
 				for (int event = bucket.getEntityStart(entity); event < end; event++) {
-					if (!bucket.has(event, column)) {
+
+					final CDateRange range = validityDate.getValidityDate(event, bucket);
+
+					if(range == null) {
 						continue;
 					}
 
-					final CDateRange range = bucket.getAsDateRange(event, column);
-
 					final int minValue = range.getMinValue();
 
 					min = Math.min(min, minValue);
@@ -192,7 +186,6 @@ private static Map<String, CDateRange> calculateEntityDateIndices(Bucket bucket)
 				final CDateRange span = CDateRange.of(min, max);
 
 				spans.compute(entity, (ignored, current) -> current == null ? span : current.span(span));
-
 			}
 		}
 

backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.query.resultinfo.printers.common;
 
 import java.util.Collection;
+import java.util.Objects;
 import java.util.StringJoiner;
 
 import com.bakdata.conquery.models.config.LocaleConfig;
@@ -24,7 +25,7 @@ public String apply(@NotNull Collection<T> f) {
 				continue;
 			}
 
-			joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString()));
+			joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj))));
 		}
 		return joiner.toString();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java

@@ -1,6 +1,5 @@
 package com.bakdata.conquery.sql.conversion.cqelement;
 
-import static com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider.SQL_UNIT_SEPARATOR;
 import static org.jooq.impl.DSL.*;
 
 import java.util.ArrayList;
@@ -24,9 +23,11 @@
 import com.google.common.base.Preconditions;
 import org.jooq.Field;
 import org.jooq.Name;
+import org.jooq.Nullability;
 import org.jooq.Record;
 import org.jooq.Table;
 import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
 
 public class CQExternalConverter implements NodeConverter<CQExternal> {
 
@@ -41,8 +42,8 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr
 			List<QueryStep> rowSelects = createRowSelects(entry, functionProvider);
 			unions.addAll(rowSelects);
 		}
-		Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal");
-		QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList());
+		Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal");
+		QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation());
 
 		Optional<ColumnDateRange> maybeValidityDate = allStep.getSelects().getValidityDate();
 
@@ -65,25 +66,22 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr
 	 * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set.
 	 */
 	private static List<QueryStep> createRowSelects(Map.Entry<String, CDateSet> entry, SqlFunctionProvider functionProvider) {
-		SqlIdColumns ids = createIdSelect(entry);
+		SqlIdColumns ids = createIdSelect(entry, functionProvider);
 		List<ColumnDateRange> validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN);
 		return validityDateEntries.stream()
 								  .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider))
 								  .toList();
 	}
 
-	private static SqlIdColumns createIdSelect(Map.Entry<String, CDateSet> entry) {
-		Field<Object> primaryColumn = DSL.inline(entry.getKey()).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias());
+	private static SqlIdColumns createIdSelect(Map.Entry<String, CDateSet> entry, SqlFunctionProvider functionProvider) {
+		Field<String> primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias());
 		return new SqlIdColumns(primaryColumn);
 	}
 
-	private static FieldWrapper<?> createExtraColumnValue(Map.Entry<String, List<String>> extraEntry) {
-		String extraValues = extraEntry.getValue().stream()
-									   .map(DSL::val)
-									   .map(Field::toString)
-									   .collect(Collectors.joining(SQL_UNIT_SEPARATOR));
+	private static FieldWrapper<?> createExtraColumnValue(Map.Entry<String, List<String>> extraEntry, SqlFunctionProvider functionProvider) {
+		Field<?> extraValues = functionProvider.asArrayRepr(extraEntry.getValue());
 		final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE));
-		final Field<?> withAlias = field(extraValues).as(alias);
+		final Field<?> withAlias = extraValues.as(alias);
 		return new FieldWrapper<>(withAlias);
 	}
 
@@ -143,24 +141,23 @@ public ConversionContext convert(CQExternal external, ConversionContext context)
 		QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context);
 		ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte);
 
-
 		if (!external.isWithExtras()) {
 			return withExternalIdCte;
 		}
 
-		QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider);
+		QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider, context);
 		return withExternalIdCte.withExternalExtras(externalExtrasCte);
 	}
 
-	private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider) {
+	private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) {
 		List<QueryStep> unions = new ArrayList<>();
 		for (Map.Entry<String, CDateSet> entry : external.getValuesResolved().entrySet()) {
 			List<Map.Entry<String, List<String>>> extrasForId = external.getExtrasForId(entry.getKey());
 			QueryStep rowSelects = createRowSelects(entry, extrasForId, functionProvider);
 			unions.add(rowSelects);
 		}
 		Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal");
-		return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList());
+		return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation());
 	}
 
 	/**
@@ -171,8 +168,8 @@ private QueryStep createRowSelects(
 			List<Map.Entry<String, List<String>>> extra,
 			SqlFunctionProvider functionProvider
 	) {
-		SqlIdColumns ids = createIdSelect(entry);
-		List<SqlSelect> extraSelects = extra.stream().map(CQExternalConverter::createExtraColumnValue).collect(Collectors.toList());
+		SqlIdColumns ids = createIdSelect(entry, functionProvider);
+		List<SqlSelect> extraSelects = extra.stream().map((Map.Entry<String, List<String>> extraEntry) -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList());
 		return createExtraRowSelect(ids, extraSelects, functionProvider);
 	}
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java

@@ -27,7 +27,7 @@ public Class<? extends CQYes> getConversionClass() {
 	public ConversionContext convert(CQYes cqYes, ConversionContext context) {
 
 		ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn();
-		Field<Object> primaryColumn = field(name(primaryColumnConfig.getField()));
+		Field<String> primaryColumn = field(name(primaryColumnConfig.getField()), String.class);
 		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
 
 		Selects selects = Selects.builder().ids(ids)

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java

@@ -1,5 +1,8 @@
 package com.bakdata.conquery.sql.conversion.cqelement.aggregation;
 
+import static org.jooq.impl.DSL.*;
+import static org.jooq.impl.DSL.field;
+
 import java.sql.Date;
 import java.util.List;
 import java.util.Optional;
@@ -57,14 +60,14 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced
 		SqlFunctionProvider functionProvider = context.getFunctionProvider();
 		ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get();
 
-		Field<Date> rangeStart = DSL.coalesce(
+		Field<Date> rangeStart = coalesce(
 				QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMinDateExpression())
+				functionProvider.getMinDateExpression()
 		).as(DateAggregationCte.RANGE_START);
 
-		Field<Date> rangeEnd = DSL.coalesce(
+		Field<Date> rangeEnd = coalesce(
 				QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME),
-				functionProvider.toDateField(functionProvider.getMaxDateExpression())
+				functionProvider.getMaxDateExpression()
 		).as(DateAggregationCte.RANGE_END);
 
 		return Selects.builder()
@@ -76,9 +79,9 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced
 
 	private TableOnConditionStep<Record> selfJoinWithShiftedRows(SqlIdColumns leftIds, SqlIdColumns rightIds, QueryStep rowNumberStep) {
 
-		Field<Integer> leftRowNumber = DSL.field(DSL.name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class)
-										  .plus(1);
-		Field<Integer> rightRowNumber = DSL.field(DSL.name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class);
+		Field<Integer> leftRowNumber = field(name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class)
+				.plus(1);
+		Field<Integer> rightRowNumber = field(name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class);
 
 		Condition[] joinConditions = Stream.concat(
 												   Stream.of(leftRowNumber.eq(rightRowNumber)),

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java

@@ -1,5 +1,6 @@
 package com.bakdata.conquery.sql.conversion.cqelement.aggregation;
 
+import static org.jooq.impl.DSL.*;
 import static org.jooq.impl.DSL.field;
 import static org.jooq.impl.DSL.keyword;
 
@@ -81,12 +82,12 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext
 			return baseStep;
 		}
 
-		Field<Object> maxDateRange = DSL.function(
+		Field<Object> maxDateRange = function(
 				"daterange",
 				Object.class,
-				this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()),
-				this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()),
-				DSL.inline("[]")
+				this.functionProvider.getMinDateExpression(),
+				this.functionProvider.getMaxDateExpression(),
+				inline("[]")
 		);
 
 		// see https://www.postgresql.org/docs/current/functions-range.html
@@ -126,7 +127,7 @@ public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggreg
 
 	private static String createEmptyRangeForNullValues(Field<?> field) {
 		Keyword datemultirange = keyword("datemultirange");
-		return DSL.coalesce(field("{0}::{1}", field, datemultirange), field("'{}'::{0}", datemultirange))
+		return coalesce(field("{0}::{1}", field, datemultirange), field("'{}'::{0}", datemultirange))
 				  .toString();
 	}