diff --git a/backend/pom.xml b/backend/pom.xml index b31ca86ca3..5d5b69d9f8 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -373,6 +373,11 @@ testcontainers-postgresql test + + org.testcontainers + testcontainers-clickhouse + test + org.testcontainers testcontainers-solr @@ -383,6 +388,12 @@ ngdbc 2.17.10 + + com.clickhouse + clickhouse-jdbc + 0.9.8 + all + io.prometheus simpleclient_dropwizard diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java index 74d4966d5f..8768f24c98 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -33,6 +33,7 @@ import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.Table; +import org.jooq.impl.DSL; @RequiredArgsConstructor public class SqlEntityResolver implements EntityResolver { @@ -144,7 +145,8 @@ private Map resolveIds(String[][] values, List rowIndex = field(name(ROW_INDEX), Integer.class); Field externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class); Field innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class); - Field isResolved = innerPrimaryColumn.isNotNull().as(IS_RESOLVED_ALIAS); + // Would prefer this to be `is not null`, but hana does not support that for fields + Field isResolved = case_().when(innerPrimaryColumn.isNull(), inline(false)).otherwise(inline(true)).as(IS_RESOLVED_ALIAS); Table allIdsTable = table(name(idColumns.getTable())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index c1effedb58..0bb1507f68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.models.config; import com.bakdata.conquery.sql.conversion.dialect.DialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.HanaDialectBundle; -import com.bakdata.conquery.sql.conversion.dialect.PostgreDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle; +import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -15,13 +16,8 @@ @Getter public enum Dialect { - /** - * Dialect for PostgreSQL database - */ POSTGRESQL(new PostgreDialectBundle()), - /** - * Dialect for SAP HANA database - */ + CLICKHOUSE(new ClickhouseDialectBundle()), HANA(new HanaDialectBundle()); private final DialectBundle dialectBundle; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 59cfaf3022..cf94d9953e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -78,6 +78,10 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } + if(singleExecution.getResultInfos() == null){ + return Collections.emptyList(); + } + // Save id column count to later check if xlsx dimensions are feasible idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos().size(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java index 2617629536..8900fa8d2f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/MappableSingleColumnSelect.java @@ -60,13 +60,10 @@ public MappableSingleColumnSelect(ColumnId column, @Nullable InternToExternMappe public static SingleColumnSqlSelect getSubstringSelect( Column column, Range.IntegerRange substringRange, SelectContext selectContext, String alias) { - Field field; - if (substringRange == null || substringRange.isAll()) { - field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); - } - else { - field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); + Field field = field(name(selectContext.getTables().getRootTable(), column.getName()), String.class); + + if (substringRange != null && !substringRange.isAll()) { if (substringRange.isAtLeast()) { field = substring(field, 1 + substringRange.getMin()); } @@ -79,7 +76,7 @@ else if (substringRange.isAtMost()) { } if (alias != null) { - field = field.as(alias); + field = field.as(name(alias)); } return new FieldWrapper<>(field, column.getName()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java index 9e503b3b9d..981c48e622 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/CBlock.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeCache; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; @@ -95,7 +96,7 @@ public static CBlock createCBlock(ConceptTreeConnector connector, BucketId bucke final int[][] mostSpecificChildren = calculateSpecificChildrenPaths(bucket, connector, bucketManager); //TODO Object2LongMap final Map includedConcepts = calculateConceptElementPathBloomFilter(bucket, mostSpecificChildren); - final Map entitySpans = calculateEntityDateIndices(bucket); + final Map entitySpans = calculateEntityDateIndices(bucket, connector); final CBlock cBlock = new CBlock(bucketId, connector.getId(), includedConcepts, entitySpans, mostSpecificChildren); return cBlock; @@ -147,17 +148,10 @@ private static Map calculateConceptElementPathBloomFilter(Bucket b * * @implNote This is an unrolled implementation of {@link CDateRange#span(CDateRange)}. */ - private static Map calculateEntityDateIndices(Bucket bucket) { + private static Map calculateEntityDateIndices(Bucket bucket, ConceptTreeConnector connector) { final Map spans = new HashMap<>(); - final Table table = bucket.getTable().resolve(); - - - for (Column column : table.getColumns()) { - if (!column.getType().isDateCompatible()) { - continue; - } - + for (ValidityDate validityDate : connector.getValidityDates()) { for (String entity : bucket.entities()) { final int end = bucket.getEntityEnd(entity); @@ -166,14 +160,14 @@ private static Map calculateEntityDateIndices(Bucket bucket) int max = Integer.MIN_VALUE; int min = Integer.MAX_VALUE; - for (int event = bucket.getEntityStart(entity); event < end; event++) { - if (!bucket.has(event, column)) { + + final CDateRange range = validityDate.getValidityDate(event, bucket); + + if(range == null) { continue; } - final CDateRange range = bucket.getAsDateRange(event, column); - final int minValue = range.getMinValue(); min = Math.min(min, minValue); @@ -192,7 +186,6 @@ private static Map calculateEntityDateIndices(Bucket bucket) final CDateRange span = CDateRange.of(min, max); spans.compute(entity, (ignored, current) -> current == null ? span : current.span(span)); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java index 218a37d71f..0e62677566 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/common/ListStringPrinter.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.query.resultinfo.printers.common; import java.util.Collection; +import java.util.Objects; import java.util.StringJoiner; import com.bakdata.conquery.models.config.LocaleConfig; @@ -24,7 +25,7 @@ public String apply(@NotNull Collection f) { continue; } - joiner.add(listFormat.escapeListElement(elementPrinter.apply(obj).toString())); + joiner.add(listFormat.escapeListElement(Objects.toString(elementPrinter.apply(obj)))); } return joiner.toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index ea148c4583..147fae21da 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.cqelement; -import static com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider.SQL_UNIT_SEPARATOR; import static org.jooq.impl.DSL.*; import java.util.ArrayList; @@ -24,9 +23,11 @@ import com.google.common.base.Preconditions; import org.jooq.Field; import org.jooq.Name; +import org.jooq.Nullability; import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; public class CQExternalConverter implements NodeConverter { @@ -41,8 +42,8 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr List rowSelects = createRowSelects(entry, functionProvider); unions.addAll(rowSelects); } - Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList()); + Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 resolved row when converting a CQExternal"); + QueryStep allStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_IDS_CTE_NAME, Collections.emptyList(), context.isNegation()); Optional maybeValidityDate = allStep.getSelects().getValidityDate(); @@ -65,25 +66,22 @@ private static QueryStep createExternalIdsCte(CQExternal external, SqlFunctionPr * 1 row per ID is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. */ private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { - SqlIdColumns ids = createIdSelect(entry); + SqlIdColumns ids = createIdSelect(entry, functionProvider); List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); return validityDateEntries.stream() .map(validityDateEntry -> createIdRowSelect(ids, validityDateEntry, functionProvider)) .toList(); } - private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.inline(entry.getKey()).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + private static SqlIdColumns createIdSelect(Map.Entry entry, SqlFunctionProvider functionProvider) { + Field primaryColumn = functionProvider.externalId(entry.getKey()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); return new SqlIdColumns(primaryColumn); } - private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry) { - String extraValues = extraEntry.getValue().stream() - .map(DSL::val) - .map(Field::toString) - .collect(Collectors.joining(SQL_UNIT_SEPARATOR)); + private static FieldWrapper createExtraColumnValue(Map.Entry> extraEntry, SqlFunctionProvider functionProvider) { + Field extraValues = functionProvider.asArrayRepr(extraEntry.getValue()); final Name alias = name(extraEntry.getKey().replace(WHITESPACE, UNDERSCORE)); - final Field withAlias = field(extraValues).as(alias); + final Field withAlias = extraValues.as(alias); return new FieldWrapper<>(withAlias); } @@ -143,16 +141,15 @@ public ConversionContext convert(CQExternal external, ConversionContext context) QueryStep externalIdsCte = createExternalIdsCte(external, functionProvider, context); ConversionContext withExternalIdCte = context.withQueryStep(externalIdsCte); - if (!external.isWithExtras()) { return withExternalIdCte; } - QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider); + QueryStep externalExtrasCte = createExternalExtrasCte(external, functionProvider, context); return withExternalIdCte.withExternalExtras(externalExtrasCte); } - private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider) { + private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvider functionProvider, ConversionContext context) { List unions = new ArrayList<>(); for (Map.Entry entry : external.getValuesResolved().entrySet()) { List>> extrasForId = external.getExtrasForId(entry.getKey()); @@ -160,7 +157,7 @@ private QueryStep createExternalExtrasCte(CQExternal external, SqlFunctionProvid unions.add(rowSelects); } Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList()); + return QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_EXTRAS_CTE_NAME, Collections.emptyList(), context.isNegation()); } /** @@ -171,8 +168,8 @@ private QueryStep createRowSelects( List>> extra, SqlFunctionProvider functionProvider ) { - SqlIdColumns ids = createIdSelect(entry); - List extraSelects = extra.stream().map(CQExternalConverter::createExtraColumnValue).collect(Collectors.toList()); + SqlIdColumns ids = createIdSelect(entry, functionProvider); + List extraSelects = extra.stream().map((Map.Entry> extraEntry) -> createExtraColumnValue(extraEntry, functionProvider)).collect(Collectors.toList()); return createExtraRowSelect(ids, extraSelects, functionProvider); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index eb6c8a234f..f45cba7d1b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -27,7 +27,7 @@ public Class getConversionClass() { public ConversionContext convert(CQYes cqYes, ConversionContext context) { ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field primaryColumn = field(name(primaryColumnConfig.getField())); + Field primaryColumn = field(name(primaryColumnConfig.getField()), String.class); SqlIdColumns ids = new SqlIdColumns(primaryColumn); Selects selects = Selects.builder().ids(ids) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 30ef3a59b3..abd998364e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; + import java.sql.Date; import java.util.List; import java.util.Optional; @@ -57,14 +60,14 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get(); - Field rangeStart = DSL.coalesce( + Field rangeStart = coalesce( QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMinDateExpression()) + functionProvider.getMinDateExpression() ).as(DateAggregationCte.RANGE_START); - Field rangeEnd = DSL.coalesce( + Field rangeEnd = coalesce( QualifyingUtil.qualify(validityDate.getStart(), ROWS_RIGHT_TABLE_NAME), - functionProvider.toDateField(functionProvider.getMaxDateExpression()) + functionProvider.getMaxDateExpression() ).as(DateAggregationCte.RANGE_END); return Selects.builder() @@ -76,9 +79,9 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftIds, SqlIdColumns rightIds, QueryStep rowNumberStep) { - Field leftRowNumber = DSL.field(DSL.name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) - .plus(1); - Field rightRowNumber = DSL.field(DSL.name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); + Field leftRowNumber = field(name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) + .plus(1); + Field rightRowNumber = field(name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); Condition[] joinConditions = Stream.concat( Stream.of(leftRowNumber.eq(rightRowNumber)), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java index 08e74dfcf8..43bb14e36a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; +import static org.jooq.impl.DSL.*; import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.keyword; @@ -81,12 +82,12 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, ConversionContext return baseStep; } - Field maxDateRange = DSL.function( + Field maxDateRange = function( "daterange", Object.class, - this.functionProvider.toDateField(this.functionProvider.getMinDateExpression()), - this.functionProvider.toDateField(this.functionProvider.getMaxDateExpression()), - DSL.inline("[]") + this.functionProvider.getMinDateExpression(), + this.functionProvider.getMaxDateExpression(), + inline("[]") ); // see https://www.postgresql.org/docs/current/functions-range.html @@ -126,7 +127,7 @@ public ColumnDateRange getAggregatedValidityDate(DateAggregationDates dateAggreg private static String createEmptyRangeForNullValues(Field field) { Keyword datemultirange = keyword("datemultirange"); - return DSL.coalesce(field("{0}::{1}", field, datemultirange), field("'{}'::{0}", datemultirange)) + return coalesce(field("{0}::{1}", field, datemultirange), field("'{}'::{0}", datemultirange)) .toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 17bc77eefe..efad103752 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import static org.jooq.impl.DSL.*; + import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -40,10 +42,10 @@ import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; +import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; import org.jooq.TableLike; -import org.jooq.impl.DSL; public class CQConceptConverter implements NodeConverter { @@ -69,11 +71,11 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep SelectContext selectContext = SelectContext.create(ids, validityDate, universalTables, context); List converted = cqConcept.getSelects().stream() - .map(selectId -> { + .map(selectId -> { Select select = selectId.resolve(); - return select.createConverter().conceptSelect(select, selectContext); + return context.getDialectBundle().getSelectConverter(select).conceptSelect(select, selectContext); }) - .toList(); + .toList(); List queriesToJoin = new ArrayList<>(); queriesToJoin.add(predecessor); @@ -88,16 +90,16 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep List allConceptSelects = Stream.concat( converted.stream().flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()), // aggregate special selects (e.g. Exists) - predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) + predecessor.getQualifiedSelects().getSqlSelects().stream().map(SqlSelect::connectorAggregate) ) - .toList(); + .toList(); Selects finalSelects = Selects.builder() - .ids(ids) - .stratificationDate(predecessorSelects.getStratificationDate()) - .validityDate(validityDate) - .sqlSelects(allConceptSelects) - .build(); + .ids(ids) + .stratificationDate(predecessorSelects.getStratificationDate()) + .validityDate(validityDate) + .sqlSelects(allConceptSelects) + .build(); TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.INNER_JOIN, context); @@ -106,28 +108,28 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep Stream.concat( finalSelects.nonExplicitSelects().stream(), finalSelects.getSqlSelects().stream() - .filter(Predicate.not(SqlSelect::isUniversal)) - .flatMap(sqlSelect -> sqlSelect.toFields().stream()) + .filter(Predicate.not(SqlSelect::isUniversal)) + .flatMap(sqlSelect -> sqlSelect.toFields().stream()) ).toList(); return QueryStep.builder() - .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) - .selects(finalSelects) - .fromTable(joinedTable) - .groupBy(groupByFields) - .predecessors(queriesToJoin) - .negate(context.isNegation()) - .build(); + .cteName(universalTables.cteName(ConceptCteStep.UNIVERSAL_SELECTS)) + .selects(finalSelects) + .fromTable(joinedTable) + .groupBy(groupByFields) + .predecessors(queriesToJoin) + .negate(context.isNegation()) + .build(); } public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getDefaultPrimaryColumn()); if (cqConcept.isExcludeFromSecondaryId() - || conversionContext.getSecondaryIdDescription() == null - || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) + || conversionContext.getSecondaryIdDescription() == null + || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription().getId()) ) { return new SqlIdColumns(primaryColumn).withAlias(); } @@ -142,13 +144,12 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv ) ); - Field secondaryId = DSL.field(DSL.name(table.getName(), secondaryIdColumn.getName())); + Field secondaryId = field(name(table.getName(), secondaryIdColumn.getName()), String.class); return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); } - private static Optional convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) { + private static ColumnDateRange convertValidityDate(String connectorLabel, ConversionContext context, ValidityDate validityDate) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); - ValidityDate validityDate = cqTable.findValidityDate(); ColumnDateRange sqlValidityDate; boolean hasValidityDate = validityDate != null; @@ -167,28 +168,39 @@ private static Optional convertValidityDate(CQTable cqTable, St sqlValidityDate = functionProvider.forCDateRange(context.getDateRestrictionRange()); } else { - return Optional.empty(); + sqlValidityDate = functionProvider.allRange(); } } - return Optional.of(sqlValidityDate.asValidityDateRange(connectorLabel)); + return sqlValidityDate.asValidityDateRange(connectorLabel); } - private static Optional collectConditionFilters(List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { - return collectConditions(conceptElements, cqTable, functionProvider) - .stream() - .reduce(WhereCondition::and) - .map(whereCondition -> new SqlFilters( - ConnectorSqlSelects.none(), - WhereClauses.builder().preprocessingCondition(whereCondition).build() - )); + private static SqlFilters collectConditionFilters( + List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { + List conditions = new ArrayList<>(); + conditions.addAll(collectConditions(conceptElements, cqTable, functionProvider)); + + ValidityDate validityDate = cqTable.findValidityDate(); + Condition validityDateFilter = noCondition(); + if (validityDate != null) { + validityDateFilter = functionProvider.isNotEmptyValidityDate(validityDate); + } + + return new SqlFilters(ConnectorSqlSelects.none(), + WhereClauses.builder() + .preprocessingConditions(conditions) + .preprocessingCondition(ConditionUtil.wrap(validityDateFilter)) + .build() + ); } private static List collectConditions(List> conceptElements, CQTable cqTable, SqlFunctionProvider functionProvider) { List conditions = new ArrayList<>(); + convertConnectorCondition(cqTable, functionProvider).ifPresent(conditions::add); + for (ConceptElement conceptElement : conceptElements) { collectConditions(cqTable, conceptElement, functionProvider) .reduce(WhereCondition::and) @@ -216,41 +228,41 @@ private static Optional convertConnectorCondition(CQTable cqTabl final Connector connector = cqTable.getConnector().resolve(); return Optional.ofNullable(connector.getCondition()) - .map(condition -> condition.convertToSqlCondition(CTConditionContext.create(connector, functionProvider))); + .map(condition -> condition.convertToSqlCondition(CTConditionContext.create(connector, functionProvider))); } private static SqlFilters dateRestrictionFilter(ConversionContext context, ColumnDateRange validityDate) { List dateRestrictionSelects = new ArrayList<>(); List conditions = new ArrayList<>(); - - conditions.add(ConditionUtil.wrap(validityDate.isNotEmpty())); + SqlFunctionProvider functionProvider = context.getFunctionProvider(); if (context.getDateRestrictionRange() != null) { - SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange dateRestriction = functionProvider.forCDateRange(context.getDateRestrictionRange()).as(SharedAliases.DATE_RESTRICTION.getAlias()); conditions.add(ConditionUtil.wrap(functionProvider.dateRestriction(dateRestriction, validityDate))); dateRestrictionSelects.addAll(dateRestriction.toFields().stream() - .map(FieldWrapper::new) - .toList()); + .map(FieldWrapper::new) + .toList()); } return new SqlFilters( ConnectorSqlSelects.builder().preprocessingSelects(dateRestrictionSelects).build(), WhereClauses.builder() - .eventFilters(conditions) - .build() + .eventFilters(conditions) + .build() ); } private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { + + return cqConcept.getSelects().stream() - .map(SelectId::resolve) - .filter(select -> select instanceof ConceptColumnSelect) - .findFirst() - .map(select -> select.createConverter().connectorSelect(select, selectContext)) - .orElse(ConnectorSqlSelects.none()); + .map(SelectId::resolve) + .filter(select -> select instanceof ConceptColumnSelect) + .findFirst() + .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .orElse(ConnectorSqlSelects.none()); } @Override @@ -263,8 +275,8 @@ public ConversionContext convert(CQConcept cqConcept, ConversionContext context) TablePath tablePath = new TablePath(cqConcept, context); List convertedCQTables = cqConcept.getTables().stream() - .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) - .toList(); + .flatMap(cqTable -> convertCqTable(tablePath, cqConcept, cqTable, context).stream()) + .toList(); QueryStep joinedStep = QueryStepJoiner.joinSteps(convertedCQTables, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.MERGE, context); QueryStep lastConceptStep = finishConceptConversion(joinedStep, cqConcept, tablePath, context); @@ -289,39 +301,40 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); ConnectorSqlTables connectorTables = tablePath.getConnectorTables(cqTable); - Optional tablesValidityDate = convertValidityDate(cqTable, connectorTables.getLabel(), conversionContext); + ColumnDateRange tablesValidityDate = convertValidityDate(connectorTables.getLabel(), conversionContext, cqTable.findValidityDate()); // convert filters SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); List allSqlFiltersForTable = new ArrayList<>(); + cqTable.getFilters().stream() - .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) - .forEach(allSqlFiltersForTable::add); - collectConditionFilters(cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(), cqTable, functionProvider).ifPresent( - allSqlFiltersForTable::add); - if (tablesValidityDate.isPresent()) { - allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate.get())); - } + .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) + .forEach(allSqlFiltersForTable::add); + + List> conceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); + allSqlFiltersForTable.add(collectConditionFilters(conceptElements, cqTable, functionProvider)); + + allSqlFiltersForTable.add(dateRestrictionFilter(conversionContext, tablesValidityDate)); // convert selects - SelectContext selectContext = SelectContext.create(ids, tablesValidityDate, connectorTables, conversionContext); + SelectContext selectContext = SelectContext.create(ids, Optional.of(tablesValidityDate), connectorTables, conversionContext); List allSelectsForTable = new ArrayList<>(); ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); allSelectsForTable.add(conceptColumnSelect); cqTable.getSelects() - .stream() - .map(SelectId::resolve) - .map(select -> select.createConverter().connectorSelect(select, selectContext)) - .forEach(allSelectsForTable::add); + .stream() + .map(SelectId::resolve) + .map(select -> selectContext.getDialectBundle().getSelectConverter(select).connectorSelect(select, selectContext)) + .forEach(allSelectsForTable::add); return CQTableContext.builder() - .ids(ids) - .validityDate(tablesValidityDate) - .sqlSelects(allSelectsForTable) - .sqlFilters(allSqlFiltersForTable) - .connectorTables(connectorTables) - .conversionContext(conversionContext) - .build(); + .ids(ids) + .validityDate(tablesValidityDate) + .sqlSelects(allSelectsForTable) + .sqlFilters(allSqlFiltersForTable) + .connectorTables(connectorTables) + .conversionContext(conversionContext) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 7c83306b23..de565ccb5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -24,7 +24,7 @@ class CQTableContext implements Context { SqlIdColumns ids; - Optional validityDate; + ColumnDateRange validityDate; List sqlSelects; List sqlFilters; ConnectorSqlTables connectorTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index da826be3a0..b2f6718dfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -58,7 +58,7 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { // validity date aggregation Optional validityDate; - if (tableContext.getValidityDate().isEmpty() || !tableContext.getConnectorTables().isWithIntervalPacking()) { + if (!tableContext.getConnectorTables().isWithIntervalPacking()) { validityDate = Optional.empty(); } else { @@ -97,7 +97,7 @@ private static IntervalPackingContext createIntervalPackingContext(CQTableContex Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); return IntervalPackingContext.builder() .ids(predcessorSelects.getIds()) - .daterange(tableContext.getValidityDate().get()) + .daterange(tableContext.getValidityDate()) .tables(tableContext.getConnectorTables()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 80f377a025..3ced541d97 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -32,7 +33,7 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { Selects preprocessingSelects = Selects.builder() .ids(tableContext.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .sqlSelects(forPreprocessing) .build(); // all where clauses that don't require any preprocessing (connector/child conditions) @@ -81,7 +82,7 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable( Selects selects = Selects.builder() .ids(stratificationSelects.getIds()) - .validityDate(tableContext.getValidityDate()) + .validityDate(Optional.of(tableContext.getValidityDate())) .stratificationDate(stratificationSelects.getStratificationDate()) .sqlSelects(preprocessingSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java index e66402d491..adf86a157b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/DialectBundle.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -7,6 +8,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.Converter; @@ -20,6 +22,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.forms.StratificationFunctions; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; @@ -34,7 +37,6 @@ import org.jooq.Field; import org.jooq.SQLDialect; -//TODO unify with com.bakdata.conquery.models.config.Dialect public interface DialectBundle { private static > List customize(List defaults, List substitutes) { @@ -104,4 +106,17 @@ default List> getDefaultNodeConverters(DSLCon ); } + default Map, ? extends SelectConverter> getSelectConverterOverrides(){ + return Collections.emptyMap(); + } + + default SelectConverter maybeOverride = (SelectConverter