diff --git a/pom.xml b/pom.xml index dc740a34..c1a41275 100644 --- a/pom.xml +++ b/pom.xml @@ -20,8 +20,9 @@ v6.4.1 1.0.0 - 8.0.28 - 3.13.0 + 3.13.0 + 0.6.4 + 1.8.0 2.10.0 2.14.0 @@ -37,7 +38,7 @@ 4.3 1.1.0 1.18.3 - 1.19.8 + 1.21.4 @@ -47,12 +48,22 @@ maf ${maf.version} + + com.clickhouse + clickhouse-jdbc + ${clickhouse_jdbc.version} + + + org.lz4 + lz4-java + ${lz4.version} + org.apache.commons commons-lang3 - ${mysql_commons_lang3.version} + ${commons_lang3.version} org.apache.commons @@ -170,11 +181,6 @@ byte-buddy ${byte_buddy.version} - - mysql - mysql-connector-java - ${mysql_driver.version} - org.skyscreamer jsonassert @@ -183,7 +189,7 @@ org.testcontainers - mysql + clickhouse ${testcontainers.version} test diff --git a/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseAutoIncrement.java b/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseAutoIncrement.java new file mode 100644 index 00000000..7c2d0e87 --- /dev/null +++ b/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseAutoIncrement.java @@ -0,0 +1,220 @@ +package org.mskcc.cbio.portal.dao; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Sequence manager that keeps counters in memory and persists the high-water mark + * in ClickHouse on shutdown so ids are never reused even if rows are deleted. + */ +public final class ClickHouseAutoIncrement { + + private static final Logger LOG = LoggerFactory.getLogger(ClickHouseAutoIncrement.class); + + private ClickHouseAutoIncrement() { + } + + private static final class SequenceConfig { + private final String tableName; + private final String columnName; + + private SequenceConfig(String tableName, String columnName) { + this.tableName = tableName; + this.columnName = columnName; + } + } + + private static final Map CONFIG = new ConcurrentHashMap<>(); + private static final Map COUNTERS = new ConcurrentHashMap<>(); + private static final String SEQUENCE_TABLE = "cbioportal_sequence_state"; + private static final AtomicBoolean SEQUENCE_TABLE_READY = new AtomicBoolean(false); + private static final AtomicBoolean SHUTDOWN_HOOK_REGISTERED = new AtomicBoolean(false); + + static { + register("seq_reference_genome", "reference_genome", "reference_genome_id"); + register("seq_cancer_study", "cancer_study", "cancer_study_id"); + register("seq_patient", "patient", "internal_id"); + register("seq_sample", "sample", "internal_id"); + register("seq_sample_list", "sample_list", "list_id"); + register("seq_genetic_entity", "genetic_entity", "id"); + register("seq_geneset", "geneset", "id"); + register("seq_geneset_hierarchy_node", "geneset_hierarchy_node", "node_id"); + register("seq_generic_entity_properties", "generic_entity_properties", "id"); + register("seq_genetic_profile", "genetic_profile", "genetic_profile_id"); + register("seq_gene_panel", "gene_panel", "internal_id"); + register("seq_structural_variant", "structural_variant", "internal_id"); + register("seq_mutation_event", "mutation_event", "mutation_event_id"); + register("seq_gistic", "gistic", "gistic_roi_id"); + register("seq_cna_event", "cna_event", "cna_event_id"); + register("seq_copy_number_seg", "copy_number_seg", "seg_id"); + register("seq_copy_number_seg_file", "copy_number_seg_file", "seg_file_id"); + register("seq_clinical_event", "clinical_event", "clinical_event_id"); + registerShutdownHook(); + } + + private static void register(String sequenceName, String tableName, String columnName) { + CONFIG.put(sequenceName, new SequenceConfig(tableName, columnName.toLowerCase(Locale.ROOT))); + } + + private static void registerShutdownHook() { + if (!SHUTDOWN_HOOK_REGISTERED.compareAndSet(false, true)) { + return; + } + Runtime.getRuntime().addShutdownHook( + new Thread(ClickHouseAutoIncrement::persistAllSequences, "clickhouse-sequence-persist")); + } + + public static long nextId(String sequenceName) throws DaoException { + SequenceConfig config = CONFIG.get(sequenceName); + if (config == null) { + throw new DaoException("Unknown sequence: " + sequenceName); + } + AtomicLong counter = COUNTERS.get(sequenceName); + if (counter == null) { + synchronized (ClickHouseAutoIncrement.class) { + counter = COUNTERS.get(sequenceName); + if (counter == null) { + long current = initializeCounter(sequenceName, config); + counter = new AtomicLong(current); + COUNTERS.put(sequenceName, counter); + } + } + } + return counter.incrementAndGet(); + } + + private static long initializeCounter(String sequenceName, SequenceConfig config) throws DaoException { + ensureSequenceTableExists(); + long persisted = fetchPersistedMax(sequenceName); + long tableMax = fetchCurrentMax(config); + long current = Math.max(persisted, tableMax); + if (current != persisted) { + persistLastValue(sequenceName, current); + } + return current; + } + + private static void ensureSequenceTableExists() throws DaoException { + if (SEQUENCE_TABLE_READY.get()) { + return; + } + synchronized (ClickHouseAutoIncrement.class) { + if (SEQUENCE_TABLE_READY.get()) { + return; + } + Connection con = null; + PreparedStatement stmt = null; + try { + con = JdbcUtil.getDbConnection(ClickHouseAutoIncrement.class); + stmt = con.prepareStatement( + "CREATE TABLE IF NOT EXISTS " + SEQUENCE_TABLE + " (" + + "sequence_name String, " + + "last_value UInt64" + + ") ENGINE = ReplacingMergeTree " + + "ORDER BY sequence_name"); + stmt.execute(); + SEQUENCE_TABLE_READY.set(true); + } catch (SQLException ex) { + throw new DaoException(ex); + } finally { + JdbcUtil.closeAll(ClickHouseAutoIncrement.class, con, stmt, null); + } + } + } + + private static long fetchPersistedMax(String sequenceName) throws DaoException { + Connection con = null; + PreparedStatement stmt = null; + ResultSet rs = null; + try { + con = JdbcUtil.getDbConnection(ClickHouseAutoIncrement.class); + stmt = con.prepareStatement( + "SELECT max(last_value) FROM " + SEQUENCE_TABLE + " WHERE sequence_name = ?"); + stmt.setString(1, sequenceName); + rs = stmt.executeQuery(); + if (rs.next()) { + return rs.getLong(1); + } + return 0L; + } catch (SQLException ex) { + throw new DaoException(ex); + } finally { + JdbcUtil.closeAll(ClickHouseAutoIncrement.class, con, stmt, rs); + } + } + + private static long fetchCurrentMax(SequenceConfig config) throws DaoException { + Connection con = null; + PreparedStatement stmt = null; + ResultSet rs = null; + try { + con = JdbcUtil.getDbConnection(ClickHouseAutoIncrement.class); + stmt = con.prepareStatement("SELECT max(" + config.columnName + ") FROM " + config.tableName); + rs = stmt.executeQuery(); + if (rs.next()) { + return rs.getLong(1); + } + return 0L; + } catch (SQLException ex) { + throw new DaoException(ex); + } finally { + JdbcUtil.closeAll(ClickHouseAutoIncrement.class, con, stmt, rs); + } + } + + private static void persistLastValue(String sequenceName, long value) throws DaoException { + Connection con = null; + PreparedStatement stmt = null; + try { + con = JdbcUtil.getDbConnection(ClickHouseAutoIncrement.class); + stmt = con.prepareStatement( + "INSERT INTO " + SEQUENCE_TABLE + " (sequence_name, last_value) VALUES (?, ?)"); + stmt.setString(1, sequenceName); + stmt.setLong(2, value); + stmt.executeUpdate(); + } catch (SQLException ex) { + throw new DaoException(ex); + } finally { + JdbcUtil.closeAll(ClickHouseAutoIncrement.class, con, stmt, null); + } + } + + private static void persistAllSequences() { + if (COUNTERS.isEmpty()) { + return; + } + try { + ensureSequenceTableExists(); + } catch (DaoException ex) { + LOG.warn("Failed to ensure sequence table during shutdown persistence.", ex); + return; + } + for (Map.Entry entry : COUNTERS.entrySet()) { + long value = entry.getValue().get(); + try { + persistLastValue(entry.getKey(), value); + } catch (DaoException ex) { + LOG.warn( + "Failed to persist sequence {} with value {} during shutdown.", + entry.getKey(), + value, + ex); + } + } + } + + public static void resetCounters() { + COUNTERS.clear(); + SEQUENCE_TABLE_READY.set(false); + } + +} diff --git a/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseBulkLoader.java b/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseBulkLoader.java new file mode 100644 index 00000000..e9122125 --- /dev/null +++ b/src/main/java/org/mskcc/cbio/portal/dao/ClickHouseBulkLoader.java @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2015 Memorial Sloan-Kettering Cancer Center. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS + * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder + * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no + * obligations to provide maintenance, support, updates, enhancements or + * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be + * liable to any party for direct, indirect, special, incidental or + * consequential damages, including lost profits, arising out of the use of this + * software and its documentation, even if Memorial Sloan-Kettering Cancer + * Center has been advised of the possibility of such damage. + */ + +/* + * This file is part of cBioPortal. + * + * cBioPortal is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY, without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.mskcc.cbio.portal.dao; + +import org.mskcc.cbio.portal.dao.DaoException; +import org.mskcc.cbio.portal.util.ProgressMonitor; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Minimal bulk loader that buffers rows in memory and streams them to ClickHouse via + * TSVWithNames over the existing JDBC connection. This replaces the previous legacy + * LOAD DATA LOCAL INFILE implementation. + *

+ * The loader keeps the public API intact so that the various DAO classes do not need to + * change their interaction pattern. + */ +public class ClickHouseBulkLoader { + + private static final Map BULK_LOADERS = new LinkedHashMap<>(); + + private static boolean bulkLoad = false; + private static boolean relaxedMode = false; + + private final String tableName; + private final List pendingRecords = new ArrayList<>(); + private String[] fieldNames = null; + + private ClickHouseBulkLoader(String tableName) { + this.tableName = tableName; + } + + public static ClickHouseBulkLoader getClickHouseBulkLoader(String tableName) { + return BULK_LOADERS.computeIfAbsent(tableName, ClickHouseBulkLoader::new); + } + + public static int flushAll() throws DaoException { + int totalInserted = 0; + for (ClickHouseBulkLoader loader : BULK_LOADERS.values()) { + totalInserted += loader.flushPendingRecords(); + } + BULK_LOADERS.clear(); + return totalInserted; + } + + private int flushPendingRecords() throws DaoException { + if (pendingRecords.isEmpty()) { + return 0; + } + + final int expectedRows = pendingRecords.size(); + Connection con = null; + PreparedStatement stmt = null; + try { + con = JdbcUtil.getDbConnection(ClickHouseBulkLoader.class); + List columns = resolveColumnNames(con); + validateRecordWidths(columns.size()); + + byte[] payload = buildTsvPayload(columns); + stmt = con.prepareStatement(buildInsertStatement(columns)); + stmt.setBinaryStream(1, new ByteArrayInputStream(payload)); + + int rowsInserted = stmt.executeUpdate(); + if (rowsInserted <= 0) { + rowsInserted = expectedRows; + } + ProgressMonitor.setCurrentMessage(" --> records inserted into `" + tableName + "` table: " + rowsInserted); + + if (!relaxedMode && rowsInserted != expectedRows) { + throw new DaoException("DB Error: only " + rowsInserted + " of the " + expectedRows + + " records were inserted in `" + tableName + "`."); + } + + pendingRecords.clear(); + return rowsInserted; + } catch (SQLException | IOException exception) { + throw new DaoException(exception); + } finally { + JdbcUtil.closeAll(ClickHouseBulkLoader.class, con, stmt, null); + } + } + + private String buildInsertStatement(List columnNames) { + final String columnsClause = columnNames.isEmpty() ? "" : " (" + String.join(",", columnNames) + ")"; + return "INSERT INTO " + tableName + columnsClause + " FORMAT TSVWithNames"; + } + + private List resolveColumnNames(Connection con) throws SQLException, DaoException { + if (fieldNames != null) { + return Arrays.asList(fieldNames); + } + + try (PreparedStatement stmt = con.prepareStatement("DESCRIBE TABLE " + tableName); + ResultSet rs = stmt.executeQuery()) { + List columns = new ArrayList<>(); + while (rs.next()) { + columns.add(rs.getString("name")); + } + if (columns.isEmpty()) { + throw new DaoException("DB Error: unable to resolve columns for `" + tableName + "`."); + } + return columns; + } + } + + private void validateRecordWidths(int columnCount) throws DaoException { + for (String[] record : pendingRecords) { + if (record.length != columnCount) { + throw new DaoException("DB Error: record column count (" + record.length + ") does not match expected column count (" + + columnCount + ") for `" + tableName + "`."); + } + } + } + + private byte[] buildTsvPayload(List columnNames) throws DaoException, IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + writeRow(buffer, columnNames); + for (String[] record : pendingRecords) { + writeRow(buffer, record); + } + return buffer.toByteArray(); + } + + private void writeRow(ByteArrayOutputStream buffer, List values) throws IOException { + for (int i = 0; i < values.size(); i++) { + if (i > 0) { + buffer.write('\t'); + } + buffer.write(escapeTsvValue(values.get(i)).getBytes(StandardCharsets.UTF_8)); + } + buffer.write('\n'); + } + + private void writeRow(ByteArrayOutputStream buffer, String[] values) throws DaoException, IOException { + if (values == null) { + throw new DaoException("DB Error: encountered null record while preparing bulk insert for `" + tableName + "`."); + } + for (int i = 0; i < values.length; i++) { + if (i > 0) { + buffer.write('\t'); + } + buffer.write(escapeTsvValue(values[i]).getBytes(StandardCharsets.UTF_8)); + } + buffer.write('\n'); + } + + public void insertRecord(String... fieldValues) { + if (fieldValues.length == 0) { + return; + } + pendingRecords.add(fieldValues); + } + + public static boolean isBulkLoad() { + return bulkLoad; + } + + public static void bulkLoadOn() { + bulkLoad = true; + } + + public static void bulkLoadOff() { + bulkLoad = false; + } + + public static void relaxedModeOn() { + relaxedMode = true; + } + + public static void relaxedModeOff() { + relaxedMode = false; + } + + public void setFieldNames(String[] fieldNames) { + this.fieldNames = fieldNames; + } + + private String escapeTsvValue(String value) { + if (value == null || "\\N".equals(value)) { + return "\\N"; + } + + StringBuilder builder = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char ch = value.charAt(i); + switch (ch) { + case '\\': + builder.append("\\\\"); + break; + case '\t': + builder.append("\\t"); + break; + case '\n': + builder.append("\\n"); + break; + case '\r': + builder.append("\\r"); + break; + default: + builder.append(ch); + break; + } + } + return builder.toString(); + } +} diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoAlleleSpecificCopyNumber.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoAlleleSpecificCopyNumber.java index 100c9ed4..8cb51faa 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoAlleleSpecificCopyNumber.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoAlleleSpecificCopyNumber.java @@ -32,7 +32,7 @@ package org.mskcc.cbio.portal.dao; -import org.mskcc.cbio.portal.model.AlleleSpecificCopyNumber; +import org.mskcc.cbio.portal.model.*; /** * Data access object for Mutation table @@ -40,11 +40,11 @@ public final class DaoAlleleSpecificCopyNumber { public static int addAlleleSpecificCopyNumber(AlleleSpecificCopyNumber ascn) throws DaoException { - if (!MySQLbulkLoader.isBulkLoad()) { - throw new DaoException("You have to turn on MySQLbulkLoader in order to insert allele specific copy numbers"); + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to insert allele specific copy numbers"); } else { int result = 1; - MySQLbulkLoader.getMySQLbulkLoader("allele_specific_copy_number").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("allele_specific_copy_number").insertRecord( resolveValueToString(ascn.getMutationEventId()), resolveValueToString(ascn.getGeneticProfileId()), resolveValueToString(ascn.getSampleId()), diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoCancerStudy.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoCancerStudy.java index dc1b5fe7..5e775a8a 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoCancerStudy.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoCancerStudy.java @@ -32,23 +32,24 @@ package org.mskcc.cbio.portal.dao; +import org.apache.commons.lang3.StringUtils; +import org.mskcc.cbio.portal.model.CancerStudy; +import org.mskcc.cbio.portal.model.CancerStudyTags; +import org.mskcc.cbio.portal.model.ReferenceGenome; +import org.mskcc.cbio.portal.model.TypeOfCancer; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.StringUtils; -import org.mskcc.cbio.portal.model.CancerStudy; -import org.mskcc.cbio.portal.model.CancerStudyTags; -import org.mskcc.cbio.portal.model.ReferenceGenome; -import org.mskcc.cbio.portal.model.TypeOfCancer; /** * Analogous to and replaces the old DaoCancerType. A CancerStudy has a NAME and @@ -61,6 +62,8 @@ */ public final class DaoCancerStudy { + private static final int DELETE_BATCH_SIZE = 500; + public static enum Status { UNAVAILABLE, AVAILABLE @@ -70,6 +73,7 @@ public static enum Status { private static final Map cacheDateByInternalId = new HashMap(); private static final Map byStableId = new HashMap(); private static final Map byInternalId = new HashMap(); + private static final String CANCER_STUDY_SEQUENCE = "seq_cancer_study"; static { reCacheAll(); @@ -214,7 +218,7 @@ public static void setImportDate(Integer internalId) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCancerStudy.class); - pstmt = con.prepareStatement("UPDATE cancer_study set IMPORT_DATE = NOW() where cancer_study_id = ?"); + pstmt = con.prepareStatement("UPDATE cancer_study set import_date = now() where cancer_study_id = ?"); pstmt.setInt(1, internalId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -234,10 +238,10 @@ public static java.util.Date getImportDate(String stableCancerStudyId, Integer . try { con = JdbcUtil.getDbConnection(DaoCancerStudy.class); if (internalId.length > 0) { - pstmt = con.prepareStatement("SELECT IMPORT_DATE FROM cancer_study where cancer_study_id = ?"); + pstmt = con.prepareStatement("SELECT import_date FROM cancer_study where cancer_study_id = ?"); pstmt.setInt(1, internalId[0]); } else { - pstmt = con.prepareStatement("SELECT IMPORT_DATE FROM cancer_study where cancer_study_identifier = ?"); + pstmt = con.prepareStatement("SELECT import_date FROM cancer_study where cancer_study_identifier = ?"); pstmt.setString(1, stableCancerStudyId); } rs = pstmt.executeQuery(); @@ -304,41 +308,38 @@ public static void addCancerStudy(CancerStudy cancerStudy, boolean overwrite) th ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCancerStudy.class); + long cancerStudyId = ClickHouseAutoIncrement.nextId(CANCER_STUDY_SEQUENCE); pstmt = con.prepareStatement("INSERT INTO cancer_study " + - "( `CANCER_STUDY_IDENTIFIER`, `NAME`, " - + "`DESCRIPTION`, `PUBLIC`, `TYPE_OF_CANCER_ID`, " - + "`PMID`, `CITATION`, `GROUPS`, `STATUS`,`REFERENCE_GENOME_ID`, `IMPORT_DATE` ) VALUES (?,?,?,?,?,?,?,?,?,?,NOW())", - Statement.RETURN_GENERATED_KEYS); - pstmt.setString(1, stableId); - pstmt.setString(2, cancerStudy.getName()); - pstmt.setString(3, cancerStudy.getDescription()); - pstmt.setBoolean(4, cancerStudy.isPublicStudy()); - pstmt.setString(5, cancerStudy.getTypeOfCancerId()); - pstmt.setString(6, cancerStudy.getPmid()); - pstmt.setString(7, cancerStudy.getCitation()); + "( `cancer_study_id`, `cancer_study_identifier`, `name`, " + + "`description`, `public`, `type_of_cancer_id`, " + + "`pmid`, `citation`, `groups`, `status`,`reference_genome_id`, `import_date` ) VALUES (?,?,?,?,?,?,?,?,?,?,?,NOW())"); + pstmt.setLong(1, cancerStudyId); + pstmt.setString(2, stableId); + pstmt.setString(3, cancerStudy.getName()); + pstmt.setString(4, cancerStudy.getDescription()); + pstmt.setBoolean(5, cancerStudy.isPublicStudy()); + pstmt.setString(6, cancerStudy.getTypeOfCancerId()); + pstmt.setString(7, cancerStudy.getPmid()); + pstmt.setString(8, cancerStudy.getCitation()); Set groups = cancerStudy.getGroups(); if (groups==null) { - pstmt.setString(8, null); + pstmt.setString(9, null); } else { - pstmt.setString(8, StringUtils.join(groups, ";")); + pstmt.setString(9, StringUtils.join(groups, ";")); } //status is UNAVAILABLE until other data is loaded for this study. Once all is loaded, the //data loading process can set this to AVAILABLE: //TODO - use this field in parts of the system that build up the list of studies to display in home page: - pstmt.setInt(9, Status.UNAVAILABLE.ordinal()); + pstmt.setInt(10, Status.UNAVAILABLE.ordinal()); try { ReferenceGenome referenceGenome = DaoReferenceGenome.getReferenceGenomeByGenomeName(cancerStudy.getReferenceGenome()); - pstmt.setInt(10, referenceGenome.getReferenceGenomeId()); + pstmt.setInt(11, referenceGenome.getReferenceGenomeId()); } catch (NullPointerException e) { throw new DaoException("Unsupported reference genome"); } pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - int autoId = rs.getInt(1); - cancerStudy.setInternalId(autoId); - } + cancerStudy.setInternalId((int)cancerStudyId); cacheCancerStudy(cancerStudy, new java.util.Date()); } catch (SQLException e) { throw new DaoException(e); @@ -356,7 +357,7 @@ public static void addCancerStudyTags(CancerStudyTags cancerStudyTags) throws Da try { con = JdbcUtil.getDbConnection(DaoCancerStudy.class); pstmt = con.prepareStatement("INSERT INTO cancer_study_tags " + - "( `CANCER_STUDY_ID`,`TAGS` ) VALUES (?,?)"); + "( `cancer_study_id`,`tags` ) VALUES (?,?)"); pstmt.setInt(1, cancerStudyTags.getCancerStudyId()); pstmt.setString(2, cancerStudyTags.getTags()); pstmt.executeUpdate(); @@ -441,10 +442,8 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCancerStudy.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE cancer_study"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { @@ -488,47 +487,6 @@ public static Set getFreshGroups(int internalCancerStudyId) throws DaoEx } } - /** - * Calls deleteCancerStudyByCascade() if cancer study exists by stable id. - * @param cancerStudyStableId - * @throws DaoException - */ - public static void deleteCancerStudyByCascade(String cancerStudyStableId) throws DaoException { - CancerStudy study = getCancerStudyByStableId(cancerStudyStableId); - if (study != null) { - deleteCancerStudyByCascade(study.getInternalId()); - } - } - - /** - * Deletes a cancer study by internal id from db with foreign key constraints enforced. - * @param internalCancerStudyId - * @throws DaoException - */ - public static void deleteCancerStudyByCascade(int internalCancerStudyId) throws DaoException { - Connection con = null; - PreparedStatement pstmt = null; - ResultSet rs = null; - try { - con = JdbcUtil.getDbConnection(DaoCancerStudy.class); - pstmt = con.prepareStatement("DELETE FROM cancer_study WHERE cancer_study_id = ?"); - pstmt.setInt(1, internalCancerStudyId); - int rows = pstmt.executeUpdate(); - // throw dao exception if no rows affected by update - if (rows == 0) { - throw new DaoException("deleteCancerStudyByCascade() failed: No rows affected"); - } - removeCancerStudyFromCache(internalCancerStudyId); - } catch (SQLException e) { - throw new DaoException(e); - } finally { - JdbcUtil.closeAll(DaoCancerStudy.class, con, pstmt, rs); - } - purgeUnreferencedRecordsAfterDeletionOfStudy(); - reCacheAll(); - System.out.println("deleted study:\nID: "+internalCancerStudyId); - } - /** * Deletes the Specified Cancer Study. * This method uses a large set of database operations to progressively delete all associated @@ -545,37 +503,6 @@ public static void deleteCancerStudyByCascade(int internalCancerStudyId) throws * @deprecated */ public static void deleteCancerStudy(int internalCancerStudyId) throws DaoException { - String[] deleteStudyStatements = { - "DELETE FROM sample_cna_event WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM genetic_alteration WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM genetic_profile_samples WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM sample_profile WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM mutation WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM alteration_driver_annotation WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM mutation_count_by_keyword WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM clinical_attribute_meta WHERE CANCER_STUDY_ID=?", - "DELETE FROM resource_definition WHERE CANCER_STUDY_ID=?", - "DELETE FROM resource_study WHERE INTERNAL_ID=?", - "DELETE FROM clinical_event_data WHERE CLINICAL_EVENT_ID IN (SELECT CLINICAL_EVENT_ID FROM clinical_event WHERE PATIENT_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?))", - "DELETE FROM clinical_event WHERE PATIENT_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?)", - "DELETE FROM sample_list_list WHERE LIST_ID IN (SELECT LIST_ID FROM sample_list WHERE CANCER_STUDY_ID=?)", - "DELETE FROM clinical_sample WHERE INTERNAL_ID IN (SELECT INTERNAL_ID FROM sample WHERE PATIENT_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?))", - "DELETE FROM resource_sample WHERE INTERNAL_ID IN (SELECT INTERNAL_ID FROM sample WHERE PATIENT_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?))", - "DELETE FROM copy_number_seg WHERE CANCER_STUDY_ID=?", - "DELETE FROM copy_number_seg_file WHERE CANCER_STUDY_ID=?", - "DELETE FROM sample WHERE PATIENT_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?)", - "DELETE FROM clinical_patient WHERE INTERNAL_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?)", - "DELETE FROM resource_patient WHERE INTERNAL_ID IN (SELECT INTERNAL_ID FROM patient WHERE CANCER_STUDY_ID=?)", - "DELETE FROM patient WHERE CANCER_STUDY_ID=?", - "DELETE FROM sample_list WHERE CANCER_STUDY_ID=?", - "DELETE FROM structural_variant WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE CANCER_STUDY_ID=?)", - "DELETE FROM genetic_profile_link WHERE REFERRED_GENETIC_PROFILE_ID IN (select GENETIC_PROFILE_ID FROM genetic_profile where CANCER_STUDY_ID=?)", - "DELETE FROM genetic_profile WHERE CANCER_STUDY_ID=?", - "DELETE FROM gistic_to_gene WHERE GISTIC_ROI_ID IN (SELECT GISTIC_ROI_ID FROM gistic WHERE CANCER_STUDY_ID=?)", - "DELETE FROM gistic WHERE CANCER_STUDY_ID=?", - "DELETE FROM mut_sig WHERE CANCER_STUDY_ID=?", - "DELETE FROM cancer_study WHERE CANCER_STUDY_ID=?;" - }; Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; @@ -584,14 +511,58 @@ public static void deleteCancerStudy(int internalCancerStudyId) throws DaoExcept DaoGenericAssay.checkAndDeleteGenericAssayMetaInStudy(internalCancerStudyId); con = JdbcUtil.getDbConnection(DaoCancerStudy.class); - for (String statementString : deleteStudyStatements) { - pstmt = con.prepareStatement(statementString); - if (statementString.contains("?")) { - pstmt.setInt(1, internalCancerStudyId); - } - pstmt.executeUpdate(); - pstmt.close(); - } + List geneticProfileIds = collectIds(con, + "SELECT genetic_profile_id FROM genetic_profile WHERE cancer_study_id=?", internalCancerStudyId); + List patientIds = collectIds(con, + "SELECT internal_id FROM patient WHERE cancer_study_id=?", internalCancerStudyId); + List sampleIds = collectIds(con, + "SELECT internal_id FROM sample WHERE patient_id IN (SELECT internal_id FROM patient WHERE cancer_study_id=?)", + internalCancerStudyId); + List sampleListIds = collectIds(con, + "SELECT list_id FROM sample_list WHERE cancer_study_id=?", internalCancerStudyId); + List clinicalEventIds = collectIds(con, + "SELECT clinical_event_id FROM clinical_event WHERE patient_id IN (SELECT internal_id FROM patient WHERE cancer_study_id=?)", + internalCancerStudyId); + List gisticIds = collectIds(con, + "SELECT gistic_roi_id FROM gistic WHERE cancer_study_id=?", internalCancerStudyId); + + deleteByIds(con, "DELETE FROM sample_cna_event WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM genetic_alteration WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM genetic_profile_samples WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM sample_profile WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM mutation WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM alteration_driver_annotation WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM mutation_count_by_keyword WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM structural_variant WHERE genetic_profile_id IN ", geneticProfileIds); + deleteByIds(con, "DELETE FROM genetic_profile_link WHERE referred_genetic_profile_id IN ", geneticProfileIds); + + deleteByIds(con, "DELETE FROM gistic_to_gene WHERE gistic_roi_id IN ", gisticIds); + + deleteByIds(con, "DELETE FROM clinical_event_data WHERE clinical_event_id IN ", clinicalEventIds); + deleteByIds(con, "DELETE FROM clinical_event WHERE clinical_event_id IN ", clinicalEventIds); + + deleteByIds(con, "DELETE FROM sample_list_list WHERE list_id IN ", sampleListIds); + + deleteByIds(con, "DELETE FROM clinical_sample WHERE internal_id IN ", sampleIds); + deleteByIds(con, "DELETE FROM resource_sample WHERE internal_id IN ", sampleIds); + + deleteByIds(con, "DELETE FROM sample WHERE internal_id IN ", sampleIds); + deleteByIds(con, "DELETE FROM clinical_patient WHERE internal_id IN ", patientIds); + deleteByIds(con, "DELETE FROM resource_patient WHERE internal_id IN ", patientIds); + + deleteByStudyId(con, "DELETE FROM clinical_attribute_meta WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM resource_definition WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM resource_study WHERE internal_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM cancer_study_tags WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM copy_number_seg WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM copy_number_seg_file WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM patient WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM sample_list WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM genetic_profile WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM gistic WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM mut_sig WHERE cancer_study_id=?", internalCancerStudyId); + deleteByStudyId(con, "DELETE FROM cancer_study WHERE cancer_study_id=?", internalCancerStudyId); + removeCancerStudyFromCache(internalCancerStudyId); } catch (SQLException e) { throw new DaoException(e); @@ -609,9 +580,9 @@ public static void deleteCancerStudy(int internalCancerStudyId) throws DaoExcept */ public static void purgeUnreferencedRecordsAfterDeletionOfStudy() throws DaoException { String[] deleteStudyStatements = { - "DELETE FROM cna_event WHERE NOT EXISTS (SELECT * FROM sample_cna_event WHERE sample_cna_event.CNA_EVENT_ID = cna_event.CNA_EVENT_ID)", - "DELETE FROM mutation_event WHERE NOT EXISTS (SELECT * FROM mutation WHERE mutation.MUTATION_EVENT_ID = mutation_event.MUTATION_EVENT_ID)" - }; + "DELETE FROM cna_event WHERE cna_event_id NOT IN (SELECT DISTINCT cna_event_id FROM sample_cna_event)", + "DELETE FROM mutation_event WHERE mutation_event_id NOT IN (SELECT DISTINCT mutation_event_id FROM mutation)" + }; Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; @@ -629,23 +600,65 @@ public static void purgeUnreferencedRecordsAfterDeletionOfStudy() throws DaoExce } } + private static List collectIds(Connection con, String sql, int cancerStudyId) throws SQLException { + PreparedStatement pstmt = null; + ResultSet rs = null; + List ids = new ArrayList<>(); + try { + pstmt = con.prepareStatement(sql); + pstmt.setInt(1, cancerStudyId); + rs = pstmt.executeQuery(); + while (rs.next()) { + ids.add(rs.getInt(1)); + } + } finally { + JdbcUtil.closeAll(DaoCancerStudy.class, null, pstmt, rs); + } + return ids; + } + + private static void deleteByStudyId(Connection con, String sql, int cancerStudyId) throws SQLException { + try (PreparedStatement pstmt = con.prepareStatement(sql)) { + pstmt.setInt(1, cancerStudyId); + pstmt.executeUpdate(); + } + } + + private static void deleteByIds(Connection con, String sqlPrefix, List ids) throws SQLException { + if (ids.isEmpty()) { + return; + } + for (int start = 0; start < ids.size(); start += DELETE_BATCH_SIZE) { + int end = Math.min(start + DELETE_BATCH_SIZE, ids.size()); + List chunk = ids.subList(start, end); + String placeholders = String.join(",", Collections.nCopies(chunk.size(), "?")); + try (PreparedStatement pstmt = con.prepareStatement(sqlPrefix + "(" + placeholders + ")")) { + int idx = 1; + for (Integer id : chunk) { + pstmt.setInt(idx++, id); + } + pstmt.executeUpdate(); + } + } + } + /** * Extracts Cancer Study JDBC Results. */ private static CancerStudy extractCancerStudy(ResultSet rs) throws DaoException { try { - CancerStudy cancerStudy = new CancerStudy(rs.getString("NAME"), - rs.getString("DESCRIPTION"), - rs.getString("CANCER_STUDY_IDENTIFIER"), - rs.getString("TYPE_OF_CANCER_ID"), - rs.getBoolean("PUBLIC")); - cancerStudy.setPmid(rs.getString("PMID")); - cancerStudy.setCitation(rs.getString("CITATION")); - cancerStudy.setGroupsInUpperCase(rs.getString("GROUPS")); - cancerStudy.setInternalId(rs.getInt("CANCER_STUDY_ID")); - cancerStudy.setImportDate(rs.getDate("IMPORT_DATE")); + CancerStudy cancerStudy = new CancerStudy(rs.getString("name"), + rs.getString("description"), + rs.getString("cancer_study_identifier"), + rs.getString("type_of_cancer_id"), + rs.getBoolean("public")); + cancerStudy.setPmid(rs.getString("pmid")); + cancerStudy.setCitation(rs.getString("citation")); + cancerStudy.setGroupsInUpperCase(rs.getString("groups")); + cancerStudy.setInternalId(rs.getInt("cancer_study_id")); + cancerStudy.setImportDate(rs.getDate("import_date")); cancerStudy.setReferenceGenome(DaoReferenceGenome.getReferenceGenomeByInternalId( - rs.getInt("REFERENCE_GENOME_ID")).getGenomeName()); + rs.getInt("reference_genome_id")).getGenomeName()); return cancerStudy; } catch (SQLException e) { throw new DaoException(e); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalAttributeMeta.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalAttributeMeta.java index c9863d7a..5b8db51f 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalAttributeMeta.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalAttributeMeta.java @@ -52,13 +52,13 @@ public static int addDatum(ClinicalAttribute attr) throws DaoException { con = JdbcUtil.getDbConnection(DaoClinicalAttributeMeta.class); pstmt = con.prepareStatement ("INSERT INTO clinical_attribute_meta(" + - "`ATTR_ID`," + - "`DISPLAY_NAME`," + - "`DESCRIPTION`," + - "`DATATYPE`," + - "`PATIENT_ATTRIBUTE`," + - "`PRIORITY`," + - "`CANCER_STUDY_ID`)" + + "`attr_id`," + + "`display_name`," + + "`description`," + + "`datatype`," + + "`patient_attribute`," + + "`priority`," + + "`cancer_study_id`)" + " VALUES(?,?,?,?,?,?,?)"); pstmt.setString(1, attr.getAttrId()); pstmt.setString(2, attr.getDisplayName()); @@ -76,13 +76,13 @@ public static int addDatum(ClinicalAttribute attr) throws DaoException { } private static ClinicalAttribute unpack(ResultSet rs) throws SQLException { - return new ClinicalAttribute(rs.getString("ATTR_ID"), - rs.getString("DISPLAY_NAME"), - rs.getString("DESCRIPTION"), - rs.getString("DATATYPE"), - rs.getBoolean("PATIENT_ATTRIBUTE"), - rs.getString("PRIORITY"), - rs.getInt("CANCER_STUDY_ID")); + return new ClinicalAttribute(rs.getString("attr_id"), + rs.getString("display_name"), + rs.getString("description"), + rs.getString("datatype"), + rs.getBoolean("patient_attribute"), + rs.getString("priority"), + rs.getInt("cancer_study_id")); } public static ClinicalAttribute getDatum(String attrId, Integer cancerStudyId) throws DaoException { @@ -104,8 +104,8 @@ public static List getDatum(Collection attrIds, Integ try { con = JdbcUtil.getDbConnection(DaoClinicalAttributeMeta.class); - pstmt = con.prepareStatement("SELECT * FROM clinical_attribute_meta WHERE ATTR_ID IN ('" - + StringUtils.join(attrIds,"','")+"') AND CANCER_STUDY_ID=" + String.valueOf(cancerStudyId)); + pstmt = con.prepareStatement("SELECT * FROM clinical_attribute_meta WHERE attr_id IN ('" + + StringUtils.join(attrIds,"','")+"') AND cancer_study_id=" + String.valueOf(cancerStudyId)); rs = pstmt.executeQuery(); @@ -132,7 +132,7 @@ public static List getDatum(Collection attrIds) throw try { con = JdbcUtil.getDbConnection(DaoClinicalAttributeMeta.class); - pstmt = con.prepareStatement("SELECT * FROM clinical_attribute_meta WHERE ATTR_ID IN ('" + pstmt = con.prepareStatement("SELECT * FROM clinical_attribute_meta WHERE attr_id IN ('" + StringUtils.join(attrIds,"','")+"')"); rs = pstmt.executeQuery(); @@ -173,8 +173,8 @@ private static List getDataByCancerStudyId(int cancerStudyId) ResultSet rs = null; PreparedStatement pstmt = null; - String sql = ("SELECT DISTINCT ATTR_ID FROM clinical_attribute_meta" - + " WHERE CANCER_STUDY_ID = " + String.valueOf(cancerStudyId)); + String sql = ("SELECT DISTINCT attr_id FROM clinical_attribute_meta" + + " WHERE cancer_study_id = " + String.valueOf(cancerStudyId)); Set attrIds = new HashSet(); try { @@ -183,7 +183,7 @@ private static List getDataByCancerStudyId(int cancerStudyId) rs = pstmt.executeQuery(); while(rs.next()) { - attrIds.add(rs.getString("ATTR_ID")); + attrIds.add(rs.getString("attr_id")); } } catch (SQLException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalData.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalData.java index d00df0d1..57e6ab7d 100755 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalData.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalData.java @@ -32,8 +32,6 @@ package org.mskcc.cbio.portal.dao; -import java.sql.*; -import java.util.*; import org.apache.commons.lang3.StringUtils; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ClinicalAttribute; @@ -43,6 +41,20 @@ import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.util.InternalIdUtil; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * Data Access Object for `clinical` table * @@ -53,12 +65,12 @@ public final class DaoClinicalData { public static final String SAMPLE_ATTRIBUTES_TABLE = "clinical_sample"; public static final String PATIENT_ATTRIBUTES_TABLE = "clinical_patient"; - private static final String SAMPLE_ATTRIBUTES_INSERT = "INSERT INTO " + SAMPLE_ATTRIBUTES_TABLE + "(`INTERNAL_ID`,`ATTR_ID`,`ATTR_VALUE`) VALUES(?,?,?)"; - private static final String PATIENT_ATTRIBUTES_INSERT = "INSERT INTO " + PATIENT_ATTRIBUTES_TABLE + "(`INTERNAL_ID`,`ATTR_ID`,`ATTR_VALUE`) VALUES(?,?,?)"; + private static final String SAMPLE_ATTRIBUTES_INSERT = "INSERT INTO " + SAMPLE_ATTRIBUTES_TABLE + "(`internal_id`,`attr_id`,`attr_value`) VALUES(?,?,?)"; + private static final String PATIENT_ATTRIBUTES_INSERT = "INSERT INTO " + PATIENT_ATTRIBUTES_TABLE + "(`internal_id`,`attr_id`,`attr_value`) VALUES(?,?,?)"; - private static final String SAMPLE_ATTRIBUTES_DELETE = "DELETE FROM " + SAMPLE_ATTRIBUTES_TABLE + " WHERE `INTERNAL_ID` = ?"; + private static final String SAMPLE_ATTRIBUTES_DELETE = "DELETE FROM " + SAMPLE_ATTRIBUTES_TABLE + " WHERE `internal_id` = ?"; - private static final String PATIENT_ATTRIBUTES_DELETE = "DELETE FROM " + PATIENT_ATTRIBUTES_TABLE + " WHERE `INTERNAL_ID` = ?"; + private static final String PATIENT_ATTRIBUTES_DELETE = "DELETE FROM " + PATIENT_ATTRIBUTES_TABLE + " WHERE `internal_id` = ?"; private static final Map sampleAttributes = new HashMap(); private static final Map patientAttributes = new HashMap(); @@ -87,7 +99,7 @@ private static void cacheAttributes(String table, Map cache) pstmt = con.prepareStatement("SELECT * FROM " + table); rs = pstmt.executeQuery(); while (rs.next()) { - cache.put(rs.getString("ATTR_ID"), rs.getString("ATTR_ID")); + cache.put(rs.getString("attr_id"), rs.getString("attr_id")); } } catch (SQLException e) { @@ -113,8 +125,8 @@ public static int addPatientDatum(int internalPatientId, String attrId, String a public static int addDatum(String query, String tableName, int internalId, String attrId, String attrVal) throws DaoException { - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.getMySQLbulkLoader(tableName).insertRecord(Integer.toString(internalId), + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.getClickHouseBulkLoader(tableName).insertRecord(Integer.toString(internalId), attrId, attrVal); return 1; @@ -191,7 +203,7 @@ private static ClinicalData getDatum(int internalCancerStudyId, String table, in con = JdbcUtil.getDbConnection(DaoClinicalData.class); pstmt = con.prepareStatement("SELECT * FROM " + table + - " WHERE INTERNAL_ID=? AND ATTR_ID=?"); + " WHERE internal_id=? AND attr_id=?"); pstmt.setInt(1, internalId); pstmt.setString(2, attrId); @@ -225,7 +237,7 @@ private static List getDataByInternalIds(int internalCancerStudyId ResultSet rs = null; List clinicals = new ArrayList(); - String sql = ("SELECT * FROM " + table + " WHERE `INTERNAL_ID` IN " + + String sql = ("SELECT * FROM " + table + " WHERE `internal_id` IN " + "(" + generateIdsSql(internalIds) + ")"); try { @@ -404,9 +416,12 @@ public static void removeSampleAttributesData(Set sampleInternalIds, St Connection con = null; PreparedStatement pstmt = null; try { + if (sampleInternalIds.isEmpty()) { + return; + } con = JdbcUtil.getDbConnection(DaoClinicalData.class); pstmt = con.prepareStatement("DELETE FROM " + SAMPLE_ATTRIBUTES_TABLE - + " WHERE `ATTR_ID` = ? AND `INTERNAL_ID` IN (" + + " WHERE `attr_id` = ? AND `internal_id` IN (" + String.join(",", Collections.nCopies(sampleInternalIds.size(), "?")) + ")"); int parameterIndex = 1; @@ -452,9 +467,9 @@ private static List getDataByInternalIds(int internalCancerStudyId List clinicals = new ArrayList(); - String sql = ("SELECT * FROM " + table + " WHERE `INTERNAL_ID` IN " + + String sql = ("SELECT * FROM " + table + " WHERE `internal_id` IN " + "(" + generateIdsSql(internalIds) + ") " + - " AND ATTR_ID IN ('"+ StringUtils.join(attributeIds, "','")+"') "); + " AND attr_id IN ('"+ StringUtils.join(attributeIds, "','")+"') "); try { con = JdbcUtil.getDbConnection(DaoClinicalData.class); @@ -484,13 +499,13 @@ public static List getDataByAttributeIds(int internalCancerStudyId con = JdbcUtil.getDbConnection(DaoClinicalData.class); pstmt = con.prepareStatement("SELECT * FROM clinical_patient WHERE" + - " ATTR_ID IN ('" + StringUtils.join(attributeIds, "','") +"') "); + " attr_id IN ('" + StringUtils.join(attributeIds, "','") +"') "); List patients = getPatientIdsByCancerStudy(internalCancerStudyId); rs = pstmt.executeQuery(); while(rs.next()) { - Integer patientId = rs.getInt("INTERNAL_ID"); + Integer patientId = rs.getInt("internal_id"); if (patients.contains(patientId)) { clinicals.add(extract(PATIENT_ATTRIBUTES_TABLE, internalCancerStudyId, rs)); } @@ -508,11 +523,11 @@ public static List getDataByAttributeIds(int internalCancerStudyId private static ClinicalData extract(String table, int internalCancerStudyId, ResultSet rs) throws SQLException { // get - String stableId = getStableIdFromInternalId(table, rs.getInt("INTERNAL_ID")); + String stableId = getStableIdFromInternalId(table, rs.getInt("internal_id")); return new ClinicalData(internalCancerStudyId, stableId, - rs.getString("ATTR_ID"), - rs.getString("ATTR_VALUE")); + rs.getString("attr_id"), + rs.getString("attr_value")); } private static String getStableIdFromInternalId(String table, int internalId) @@ -664,8 +679,8 @@ private static List getIdsByAttribute(int cancerStudyId, String paramNa try{ con = JdbcUtil.getDbConnection(DaoClinicalData.class); - pstmt = con.prepareStatement ("SELECT INTERNAL_ID FROM `" + tableName + "`" - + " WHERE ATTR_ID=? AND ATTR_VALUE=?"); + pstmt = con.prepareStatement ("SELECT internal_id FROM `" + tableName + "`" + + " WHERE attr_id=? AND attr_value=?"); pstmt.setString(1, paramName); pstmt.setString(2, paramValue); rs = pstmt.executeQuery(); @@ -674,7 +689,7 @@ private static List getIdsByAttribute(int cancerStudyId, String paramNa while (rs.next()) { - ids.add(rs.getInt("INTERNAL_ID")); + ids.add(rs.getInt("internal_id")); } return ids; @@ -695,10 +710,10 @@ public static Map> getCancerTypeInfoBySamples(List s try{ con = JdbcUtil.getDbConnection(DaoClinicalData.class); pstmt = con.prepareStatement("select " + - "distinct ATTR_VALUE as attributeValue, " + - "ATTR_ID as attributeID from clinical_sample " + - "where ATTR_ID in (?, ?) and INTERNAL_ID in (" + - "select INTERNAL_ID from sample where STABLE_ID in ('" + "distinct attr_value as attributeValue, " + + "attr_id as attributeID from clinical_sample " + + "where attr_id in (?, ?) and internal_id in (" + + "select internal_id from sample where stable_id in ('" + StringUtils.join(samplesList,"','")+"'))"); pstmt.setString(1, ClinicalAttribute.CANCER_TYPE); pstmt.setString(2, ClinicalAttribute.CANCER_TYPE_DETAILED); @@ -738,4 +753,45 @@ public static void removePatientAttributesData(int internalPatientId) throws Dao JdbcUtil.closeAll(DaoClinicalData.class, con, pstmt, rs); } } + + private static List> getExistingAttributes(String tableName, List> internalPatientIdAttributes) throws DaoException { + if (internalPatientIdAttributes.isEmpty()) { + return Collections.emptyList(); + } + Connection con = null; + PreparedStatement pstmt = null; + ResultSet rs = null; + + try { + con = JdbcUtil.getDbConnection(DaoClinicalData.class); + pstmt = con.prepareStatement("SELECT internal_id, attr_id FROM `" + + tableName + "` WHERE (internal_id, attr_id) IN (" + + String.join(",", Collections.nCopies(internalPatientIdAttributes.size(), "(?, ?)")) + ")"); + int parameterIndex = 1; + for (Map.Entry entry : internalPatientIdAttributes) { + pstmt.setInt(parameterIndex++, entry.getKey()); + pstmt.setString(parameterIndex++, entry.getValue()); + } + rs = pstmt.executeQuery(); + + List> result = new ArrayList<>(); + while (rs.next()) { + result.add(Map.entry(rs.getInt("internal_id"), rs.getString("attr_id"))); + } + + return result; + } catch (SQLException e) { + throw new DaoException(e); + } finally { + JdbcUtil.closeAll(DaoClinicalData.class, con, pstmt, rs); + } + } + + public static List> getExistingPatientAttributes(List> internalPatientIdAttributes) throws DaoException { + return getExistingAttributes(PATIENT_ATTRIBUTES_TABLE, internalPatientIdAttributes); + } + + public static List> getExistingSampleAttributes(List> internalPatientIdAttributes) throws DaoException { + return getExistingAttributes(SAMPLE_ATTRIBUTES_TABLE, internalPatientIdAttributes); + } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalEvent.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalEvent.java index b7501e6f..44620f7b 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalEvent.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoClinicalEvent.java @@ -51,11 +51,11 @@ public final class DaoClinicalEvent { private DaoClinicalEvent() {} public static int addClinicalEvent(ClinicalEvent clinicalEvent) { - if (!MySQLbulkLoader.isBulkLoad()) { + if (!ClickHouseBulkLoader.isBulkLoad()) { throw new IllegalStateException("Only bulk load mode is allowed for importing clinical events"); } - MySQLbulkLoader.getMySQLbulkLoader("clinical_event").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("clinical_event").insertRecord( Long.toString(clinicalEvent.getClinicalEventId()), Integer.toString(clinicalEvent.getPatientId()), clinicalEvent.getStartDate().toString(), @@ -68,7 +68,7 @@ public static int addClinicalEvent(ClinicalEvent clinicalEvent) { private static int addClinicalEventData(ClinicalEvent clinicalEvent) { long eventId = clinicalEvent.getClinicalEventId(); for (Map.Entry entry : clinicalEvent.getEventData().entrySet()) { - MySQLbulkLoader.getMySQLbulkLoader("clinical_event_data").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("clinical_event_data").insertRecord( Long.toString(eventId), entry.getKey(), entry.getValue() @@ -92,9 +92,9 @@ public static List getClinicalEvent(int patientId, String eventTy // get events first if (eventType==null) { - pstmt = con.prepareStatement("SELECT * FROM clinical_event WHERE PATIENT_ID=?"); + pstmt = con.prepareStatement("SELECT * FROM clinical_event WHERE patient_id=?"); } else { - pstmt = con.prepareStatement("SELECT * FROM clinical_event WHERE PATIENT_ID=? AND EVENT_TYPE=?"); + pstmt = con.prepareStatement("SELECT * FROM clinical_event WHERE patient_id=? AND event_type=?"); } pstmt.setInt(1, patientId); if (eventType!=null) { @@ -112,13 +112,13 @@ public static List getClinicalEvent(int patientId, String eventTy // get data then if (!clinicalEvents.isEmpty()) { - pstmt = con.prepareStatement("SELECT * FROM clinical_event_data WHERE CLINICAL_EVENT_ID IN (" + pstmt = con.prepareStatement("SELECT * FROM clinical_event_data WHERE clinical_event_id IN (" + StringUtils.join(clinicalEvents.keySet(), ",") + ")"); rs = pstmt.executeQuery(); while (rs.next()) { - long eventId = rs.getLong("CLINICAL_EVENT_ID"); - clinicalEvents.get(eventId).addEventDatum(rs.getString("KEY"), rs.getString("VALUE")); + long eventId = rs.getLong("clinical_event_id"); + clinicalEvents.get(eventId).addEventDatum(rs.getString("key"), rs.getString("value")); } } @@ -132,11 +132,11 @@ public static List getClinicalEvent(int patientId, String eventTy private static ClinicalEvent extractClinicalEvent(ResultSet rs) throws SQLException { ClinicalEvent clinicalEvent = new ClinicalEvent(); - clinicalEvent.setClinicalEventId(rs.getLong("CLINICAL_EVENT_ID")); - clinicalEvent.setPatientId(rs.getInt("PATIENT_ID")); - clinicalEvent.setStartDate(JdbcUtil.readLongFromResultSet(rs, "START_DATE")); - clinicalEvent.setStopDate(JdbcUtil.readLongFromResultSet(rs, "STOP_DATE")); - clinicalEvent.setEventType(rs.getString("EVENT_TYPE")); + clinicalEvent.setClinicalEventId(rs.getLong("clinical_event_id")); + clinicalEvent.setPatientId(rs.getInt("patient_id")); + clinicalEvent.setStartDate(JdbcUtil.readLongFromResultSet(rs, "start_date")); + clinicalEvent.setStopDate(JdbcUtil.readLongFromResultSet(rs, "stop_date")); + clinicalEvent.setEventType(rs.getString("event_type")); return clinicalEvent; } @@ -147,7 +147,7 @@ public static long getLargestClinicalEventId() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoClinicalEvent.class); pstmt = con.prepareStatement - ("SELECT MAX(`CLINICAL_EVENT_ID`) FROM `clinical_event`"); + ("SELECT max(`clinical_event_id`) FROM `clinical_event`"); rs = pstmt.executeQuery(); return rs.next() ? rs.getLong(1) : 0; } catch (SQLException e) { @@ -170,7 +170,7 @@ public static boolean timeEventsExistForPatient(int patientId) throws DaoExcepti ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); - pstmt = con.prepareStatement("SELECT EXISTS(SELECT 1 FROM `clinical_event` WHERE `PATIENT_ID`=?)"); + pstmt = con.prepareStatement("SELECT EXISTS(SELECT 1 FROM `clinical_event` WHERE `patient_id`=?)"); pstmt.setInt(1, patientId); rs = pstmt.executeQuery(); return rs.next() && rs.getInt(1)==1; @@ -188,12 +188,12 @@ public static void deleteByCancerStudyId(int cancerStudyId) throws DaoException try { con = JdbcUtil.getDbConnection(DaoClinicalEvent.class); - pstmt = con.prepareStatement("DELETE FROM clinical_event_data WHERE CLINICAL_EVENT_ID IN " - + "(SELECT CLINICAL_EVENT_ID FROM clinical_event WHERE PATIENT_ID in (SELECT INTERNAL_ID FROM patient where CANCER_STUDY_ID=?))"); + pstmt = con.prepareStatement("DELETE FROM clinical_event_data WHERE clinical_event_id IN " + + "(SELECT clinical_event_id FROM clinical_event WHERE patient_id in (SELECT internal_id FROM patient where cancer_study_id=?))"); pstmt.setInt(1, cancerStudyId); pstmt.executeUpdate(); - pstmt = con.prepareStatement("DELETE FROM clinical_event WHERE PATIENT_ID in (SELECT INTERNAL_ID FROM patient where CANCER_STUDY_ID=?)"); + pstmt = con.prepareStatement("DELETE FROM clinical_event WHERE patient_id in (SELECT internal_id FROM patient where cancer_study_id=?)"); pstmt.setInt(1, cancerStudyId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -210,7 +210,12 @@ public static void deleteByPatientId(int patientId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoClinicalEvent.class); - pstmt = con.prepareStatement("DELETE FROM clinical_event WHERE clinical_event.PATIENT_ID = ?"); + pstmt = con.prepareStatement("DELETE FROM clinical_event_data WHERE clinical_event_id IN (" + + "SELECT clinical_event_id FROM clinical_event WHERE patient_id = ?)"); + pstmt.setInt(1, patientId); + pstmt.executeUpdate(); + pstmt.close(); + pstmt = con.prepareStatement("DELETE FROM clinical_event WHERE patient_id = ?"); pstmt.setInt(1, patientId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -226,12 +231,10 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoClinicalData.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE clinical_event_data"); pstmt.executeUpdate(); pstmt = con.prepareStatement("TRUNCATE TABLE clinical_event"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoCnaEvent.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoCnaEvent.java index fa8fa7e0..4cf0facd 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoCnaEvent.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoCnaEvent.java @@ -43,11 +43,13 @@ * @author jgao */ public final class DaoCnaEvent { + + private static final String CNA_EVENT_SEQUENCE = "seq_cna_event"; private DaoCnaEvent() {} public static void addCaseCnaEvent(CnaEvent cnaEvent, boolean newCnaEvent) throws DaoException { - if (!MySQLbulkLoader.isBulkLoad()) { - throw new DaoException("You have to turn on MySQLbulkLoader in order to insert sample_cna_event"); + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to insert sample_cna_event"); } else { long eventId = cnaEvent.getEventId(); @@ -57,7 +59,7 @@ public static void addCaseCnaEvent(CnaEvent cnaEvent, boolean newCnaEvent) throw cnaEvent.setEventId(eventId); } - MySQLbulkLoader.getMySQLbulkLoader("sample_cna_event").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("sample_cna_event").insertRecord( Long.toString(eventId), Integer.toString(cnaEvent.getSampleId()), Integer.toString(cnaEvent.getCnaProfileId()), @@ -72,8 +74,8 @@ public static void addCaseCnaEvent(CnaEvent cnaEvent, boolean newCnaEvent) throw && !cnaEvent.getDriverTiersFilter().isEmpty() && !cnaEvent.getDriverTiersFilter().toLowerCase().equals("na")) ) { - MySQLbulkLoader - .getMySQLbulkLoader("alteration_driver_annotation") + ClickHouseBulkLoader + .getClickHouseBulkLoader("alteration_driver_annotation") .insertRecord( Long.toString(eventId), Integer.toString(cnaEvent.getCnaProfileId()), @@ -97,25 +99,23 @@ public static void addCaseCnaEvent(CnaEvent cnaEvent, boolean newCnaEvent) throw private static long addCnaEventDirectly(CnaEvent cnaEvent) throws DaoException { Connection con = null; PreparedStatement pstmt = null; - ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCnaEvent.class); + long newId = ClickHouseAutoIncrement.nextId(CNA_EVENT_SEQUENCE); pstmt = con.prepareStatement ("INSERT INTO cna_event (" + - "`ENTREZ_GENE_ID`," + - "`ALTERATION` )" + - " VALUES(?,?)", Statement.RETURN_GENERATED_KEYS); - pstmt.setLong(1, cnaEvent.getEntrezGeneId()); - pstmt.setInt(2, cnaEvent.getAlteration()); + "`cna_event_id`, `entrez_gene_id`," + + "`alteration` )" + + " VALUES(?,?,?)"); + pstmt.setLong(1, newId); + pstmt.setLong(2, cnaEvent.getEntrezGeneId()); + pstmt.setInt(3, cnaEvent.getAlteration()); pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - rs.next(); - long newId = rs.getLong(1); return newId; } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoCnaEvent.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoCnaEvent.class, con, pstmt, null); } } @@ -124,21 +124,33 @@ public static void removeSampleCnaEvents(int cnaProfileId, List sampleI PreparedStatement pstmt = null; ResultSet rs = null; try { + if (sampleIds.isEmpty()) { + return; + } + String placeholders = String.join(",", Collections.nCopies(sampleIds.size(), "?")); con = JdbcUtil.getDbConnection(DaoCnaEvent.class); - pstmt = con.prepareStatement - ("DELETE sample_cna_event, alteration_driver_annotation" + - " FROM sample_cna_event" + - " LEFT JOIN alteration_driver_annotation ON alteration_driver_annotation.`ALTERATION_EVENT_ID` = sample_cna_event.`CNA_EVENT_ID`" + - " AND alteration_driver_annotation.`SAMPLE_ID` = sample_cna_event.`SAMPLE_ID`" + - " AND alteration_driver_annotation.`GENETIC_PROFILE_ID` = sample_cna_event.`GENETIC_PROFILE_ID`" + - " WHERE sample_cna_event.`GENETIC_PROFILE_ID` = ? AND sample_cna_event.`SAMPLE_ID` IN (" + - String.join(",", Collections.nCopies(sampleIds.size(), "?")) - + ")"); + pstmt = con.prepareStatement( + "DELETE FROM alteration_driver_annotation " + + "WHERE `genetic_profile_id` = ? AND `sample_id` IN (" + placeholders + ") " + + "AND `alteration_event_id` IN (SELECT `cna_event_id` FROM sample_cna_event WHERE `genetic_profile_id` = ? AND `sample_id` IN (" + placeholders + "))"); int parameterIndex = 1; pstmt.setInt(parameterIndex++, cnaProfileId); for (Integer sampleId : sampleIds) { pstmt.setInt(parameterIndex++, sampleId); } + pstmt.setInt(parameterIndex++, cnaProfileId); + for (Integer sampleId : sampleIds) { + pstmt.setInt(parameterIndex++, sampleId); + } + pstmt.executeUpdate(); + pstmt.close(); + pstmt = con.prepareStatement( + "DELETE FROM sample_cna_event WHERE `genetic_profile_id` = ? AND `sample_id` IN (" + placeholders + ")"); + parameterIndex = 1; + pstmt.setInt(parameterIndex++, cnaProfileId); + for (Integer sampleId : sampleIds) { + pstmt.setInt(parameterIndex++, sampleId); + } pstmt.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); @@ -163,15 +175,15 @@ public static Map> getSamplesWithAlterations(String concatEven try { con = JdbcUtil.getDbConnection(DaoCnaEvent.class); String sql = "SELECT * FROM sample_cna_event" - + " WHERE `CNA_EVENT_ID` IN (" + + " WHERE `cna_event_id` IN (" + concatEventIds + ")"; pstmt = con.prepareStatement(sql); Map> map = new HashMap> (); rs = pstmt.executeQuery(); while (rs.next()) { - Sample sample = DaoSample.getSampleById(rs.getInt("SAMPLE_ID")); - long eventId = rs.getLong("CNA_EVENT_ID"); + Sample sample = DaoSample.getSampleById(rs.getInt("sample_id")); + long eventId = rs.getLong("cna_event_id"); Set events = map.get(sample); if (events == null) { events = new HashSet(); @@ -196,26 +208,25 @@ public static List getCnaEvents(List sampleIds, Collection events = new ArrayList(); @@ -231,15 +242,15 @@ public static List getCnaEvents(List sampleIds, Collection getAllCnaEvents() throws DaoException { while (rs.next()) { try { CnaEvent.Event event = new CnaEvent.Event(); - event.setEventId(rs.getLong("CNA_EVENT_ID")); - event.setEntrezGeneId(rs.getLong("ENTREZ_GENE_ID")); - event.setAlteration(rs.getInt("ALTERATION")); + event.setEventId(rs.getLong("cna_event_id")); + event.setEntrezGeneId(rs.getLong("entrez_gene_id")); + event.setAlteration(rs.getShort("alteration")); events.add(event); } catch (IllegalArgumentException e) { e.printStackTrace(); @@ -287,13 +298,13 @@ public static Map> countSamplesWithCNAGenes( ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCnaEvent.class); - String sql = "SELECT `ENTREZ_GENE_ID`, `ALTERATION`, count(*)" + String sql = "SELECT `entrez_gene_id`, `alteration`, count(*)" + " FROM sample_cna_event, cna_event" - + " WHERE `GENETIC_PROFILE_ID`=" + profileId - + " and sample_cna_event.`CNA_EVENT_ID`=cna_event.`CNA_EVENT_ID`" - + " and `ENTREZ_GENE_ID` IN (" + + " WHERE `genetic_profile_id`=" + profileId + + " and sample_cna_event.`cna_event_id`=cna_event.`cna_event_id`" + + " and `entrez_gene_id` IN (" + concatEntrezGeneIds - + ") GROUP BY `ENTREZ_GENE_ID`, `ALTERATION`"; + + ") GROUP BY `entrez_gene_id`, `alteration`"; pstmt = con.prepareStatement(sql); Map> map = new HashMap>(); @@ -332,11 +343,11 @@ public static Map countSamplesWithCnaEvents(String concatEventIds ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCnaEvent.class); - String sql = "SELECT `CNA_EVENT_ID`, count(*) FROM sample_cna_event" - + " WHERE `GENETIC_PROFILE_ID`=" + profileId - + " and `CNA_EVENT_ID` IN (" + String sql = "SELECT `cna_event_id`, count(*) FROM sample_cna_event" + + " WHERE `genetic_profile_id`=" + profileId + + " and `cna_event_id` IN (" + concatEventIds - + ") GROUP BY `CNA_EVENT_ID`"; + + ") GROUP BY `cna_event_id`"; pstmt = con.prepareStatement(sql); Map map = new HashMap(); @@ -362,8 +373,8 @@ public static Set getAlteredGenes(String concatEventIds) ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCnaEvent.class); - String sql = "SELECT DISTINCT ENTREZ_GENE_ID FROM cna_event " - + "WHERE CNA_EVENT_ID in (" + String sql = "SELECT DISTINCT entrez_gene_id FROM cna_event " + + "WHERE cna_event_id in (" + concatEventIds + ")"; pstmt = con.prepareStatement(sql); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegment.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegment.java index 5f81737f..42d97192 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegment.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegment.java @@ -50,10 +50,10 @@ public final class DaoCopyNumberSegment { private DaoCopyNumberSegment() {} public static int addCopyNumberSegment(CopyNumberSegment seg) throws DaoException { - if (!MySQLbulkLoader.isBulkLoad()) { - throw new DaoException("You have to turn on MySQLbulkLoader in order to insert mutations"); + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to insert mutations"); } else { - MySQLbulkLoader.getMySQLbulkLoader("copy_number_seg").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("copy_number_seg").insertRecord( Long.toString(seg.getSegId()), Integer.toString(seg.getCancerStudyId()), Integer.toString(seg.getSampleId()), @@ -77,21 +77,29 @@ public static int addCopyNumberSegment(CopyNumberSegment seg) throws DaoExceptio */ public static void createFractionGenomeAlteredClinicalData(int cancerStudyId, Set sampleIds, boolean updateMode) throws DaoException { + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to insert Fraction Genome Altered"); + } + Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); pstmt = con.prepareStatement( - "SELECT `SAMPLE_ID`, IF((SELECT SUM(`END`-`START`) FROM copy_number_seg " + - "AS c2 WHERE c2.`CANCER_STUDY_ID` = c1.`CANCER_STUDY_ID` AND c2.`SAMPLE_ID` = c1.`SAMPLE_ID` AND " + - "ABS(c2.`SEGMENT_MEAN`) >= 0.2) IS NULL, 0, (SELECT SUM(`END`-`START`) FROM copy_number_seg " + - "AS c2 WHERE c2.`CANCER_STUDY_ID` = c1.`CANCER_STUDY_ID` AND c2.`SAMPLE_ID` = c1.`SAMPLE_ID` AND " + - "ABS(c2.`SEGMENT_MEAN`) >= 0.2) / SUM(`END`-`START`)) AS `VALUE` FROM `copy_number_seg` AS c1 , `cancer_study` " + - "WHERE c1.`CANCER_STUDY_ID` = cancer_study.`CANCER_STUDY_ID` AND cancer_study.`CANCER_STUDY_ID`=? " + - (sampleIds == null ? "" : ("AND `SAMPLE_ID` IN ("+ String.join(",", Collections.nCopies(sampleIds.size(), "?")) + ") ")) - +"GROUP BY cancer_study.`CANCER_STUDY_ID` , `SAMPLE_ID` HAVING SUM(`END`-`START`) > 0;"); + "SELECT c1.`sample_id`, " + + "CASE WHEN SUM(c1.`end` - c1.`start`) > 0 THEN " + + "ROUND(SUM(CASE WHEN ABS(c1.`segment_mean`) >= ? THEN (c1.`end` - c1.`start`) ELSE 0 END) * 1.0 / " + + "SUM(c1.`end` - c1.`start`), 4) " + + "ELSE 0 END AS `value` " + + "FROM `copy_number_seg` AS c1 " + + "INNER JOIN `cancer_study` ON c1.`cancer_study_id` = cancer_study.`cancer_study_id` " + + "WHERE cancer_study.`cancer_study_id`=? " + + (sampleIds == null ? "" : ("AND c1.`sample_id` IN (" + String.join(",", Collections.nCopies(sampleIds.size(), "?")) + ") ")) + + "GROUP BY cancer_study.`cancer_study_id`, c1.`sample_id` " + + "HAVING SUM(c1.`end` - c1.`start`) > 0"); int parameterIndex = 1; + pstmt.setDouble(parameterIndex++, FRACTION_GENOME_ALTERED_CUTOFF); pstmt.setInt(parameterIndex++, cancerStudyId); if (sampleIds != null) { for (Integer sampleId : sampleIds) { @@ -131,7 +139,7 @@ public static long getLargestId() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement - ("SELECT MAX(`SEG_ID`) FROM `copy_number_seg`"); + ("SELECT max(`seg_id`) FROM `copy_number_seg`"); rs = pstmt.executeQuery(); return rs.next() ? rs.getLong(1) : 0; } catch (SQLException e) { @@ -161,19 +169,19 @@ public static List getSegmentForSamples( con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); pstmt = con.prepareStatement ("SELECT * FROM copy_number_seg" - + " WHERE `SAMPLE_ID` IN "+ concatSampleIds - + " AND `CANCER_STUDY_ID`="+cancerStudyId); + + " WHERE `sample_id` IN "+ concatSampleIds + + " AND `cancer_study_id`="+cancerStudyId); rs = pstmt.executeQuery(); while (rs.next()) { CopyNumberSegment seg = new CopyNumberSegment( - rs.getInt("CANCER_STUDY_ID"), - rs.getInt("SAMPLE_ID"), - rs.getString("CHR"), - rs.getLong("START"), - rs.getLong("END"), - rs.getInt("NUM_PROBES"), - rs.getDouble("SEGMENT_MEAN")); - seg.setSegId(rs.getLong("SEG_ID")); + rs.getInt("cancer_study_id"), + rs.getInt("sample_id"), + rs.getString("chr"), + rs.getLong("start"), + rs.getLong("end"), + rs.getInt("num_probes"), + rs.getDouble("segment_mean")); + seg.setSegId(rs.getLong("seg_id")); segs.add(seg); } return segs; @@ -222,18 +230,18 @@ private static Map getCopyNumberAlteredLength(Collection try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); if (cutoff>0) { - sql = "SELECT `SAMPLE_ID`, SUM(`END`-`START`)" + sql = "SELECT `sample_id`, sum(`end`-`start`)" + " FROM `copy_number_seg`" - + " WHERE `CANCER_STUDY_ID`="+cancerStudyId - + " AND ABS(`SEGMENT_MEAN`)>=" + cutoff - + " AND `SAMPLE_ID` IN ('" + StringUtils.join(sampleIds,"','") +"')" - + " GROUP BY `SAMPLE_ID`"; + + " WHERE `cancer_study_id`="+cancerStudyId + + " AND abs(`segment_mean`)>=" + cutoff + + " AND `sample_id` IN ('" + StringUtils.join(sampleIds,"','") +"')" + + " GROUP BY `sample_id`"; } else { - sql = "SELECT `SAMPLE_ID`, SUM(`END`-`START`)" + sql = "SELECT `sample_id`, sum(`end`-`start`)" + " FROM `copy_number_seg`" - + " WHERE `CANCER_STUDY_ID`="+cancerStudyId - + " AND `SAMPLE_ID` IN ('" + StringUtils.join(sampleIds,"','") +"')" - + " GROUP BY `SAMPLE_ID`"; + + " WHERE `cancer_study_id`="+cancerStudyId + + " AND `sample_id` IN ('" + StringUtils.join(sampleIds,"','") +"')" + + " GROUP BY `sample_id`"; } pstmt = con.prepareStatement(sql); @@ -264,7 +272,7 @@ public static boolean segmentDataExistForCancerStudy(int cancerStudyId) throws D ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); - pstmt = con.prepareStatement("SELECT EXISTS (SELECT 1 FROM `copy_number_seg` WHERE `CANCER_STUDY_ID`=?)"); + pstmt = con.prepareStatement("SELECT EXISTS (SELECT 1 FROM `copy_number_seg` WHERE `cancer_study_id`=?)"); pstmt.setInt(1, cancerStudyId); rs = pstmt.executeQuery(); return rs.next() && rs.getInt(1)==1; @@ -289,7 +297,7 @@ public static boolean segmentDataExistForSample(int cancerStudyId, int sampleId) try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); pstmt = con.prepareStatement("SELECT EXISTS(SELECT 1 FROM `copy_number_seg`" - + " WHERE `CANCER_STUDY_ID`=? AND `SAMPLE_ID`=?"); + + " WHERE `cancer_study_id`=? AND `sample_id`=?"); pstmt.setInt(1, cancerStudyId); pstmt.setInt(2, sampleId); rs = pstmt.executeQuery(); @@ -310,8 +318,8 @@ public static void deleteSegmentDataForSamples(int cancerStudyId, Set try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); pstmt = con.prepareStatement("DELETE FROM `copy_number_seg`" + - " WHERE `CANCER_STUDY_ID`= ?" + - " AND `SAMPLE_ID` IN (" + String.join(",", Collections.nCopies(sampleIds.size(), "?")) + " WHERE `cancer_study_id`= ?" + + " AND `sample_id` IN (" + String.join(",", Collections.nCopies(sampleIds.size(), "?")) + ")"); int parameterIndex = 1; pstmt.setInt(parameterIndex++, cancerStudyId); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegmentFile.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegmentFile.java index 41e5ef3a..3bd687fd 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegmentFile.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoCopyNumberSegmentFile.java @@ -36,32 +36,30 @@ import org.mskcc.cbio.portal.model.CopyNumberSegmentFile; public final class DaoCopyNumberSegmentFile { + private static final String COPY_NUMBER_SEG_FILE_SEQUENCE = "seq_copy_number_seg_file"; private DaoCopyNumberSegmentFile() {} public static int addCopyNumberSegmentFile(CopyNumberSegmentFile copySegFile) throws DaoException { Connection con = null; PreparedStatement pstmt = null; - ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegmentFile.class); + long fileId = ClickHouseAutoIncrement.nextId(COPY_NUMBER_SEG_FILE_SEQUENCE); pstmt = con.prepareStatement - ("INSERT INTO copy_number_seg_file (`CANCER_STUDY_ID`, `REFERENCE_GENOME_ID`, `DESCRIPTION`,`FILENAME`)" - + " VALUES (?,?,?,?)", Statement.RETURN_GENERATED_KEYS); - pstmt.setInt(1, copySegFile.cancerStudyId); - pstmt.setString(2, copySegFile.referenceGenomeId.toString()); - pstmt.setString(3, copySegFile.description); - pstmt.setString(4, copySegFile.filename); + ("INSERT INTO copy_number_seg_file (`seg_file_id`, `cancer_study_id`, `reference_genome_id`, `description`,`filename`)" + + " VALUES (?,?,?,?,?)"); + pstmt.setLong(1, fileId); + pstmt.setInt(2, copySegFile.cancerStudyId); + pstmt.setString(3, copySegFile.referenceGenomeId.toString()); + pstmt.setString(4, copySegFile.description); + pstmt.setString(5, copySegFile.filename); pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - return rs.getInt(1); - } - return -1; + return (int) fileId; } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoCopyNumberSegmentFile.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoCopyNumberSegmentFile.class, con, pstmt, null); } } @@ -72,16 +70,16 @@ public static CopyNumberSegmentFile getCopyNumberSegmentFile(int cancerStudyId) ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegmentFile.class); - pstmt = con.prepareStatement("SELECT * from copy_number_seg_file WHERE `CANCER_STUDY_ID` = ?"); + pstmt = con.prepareStatement("SELECT * from copy_number_seg_file WHERE `cancer_study_id` = ?"); pstmt.setInt(1, cancerStudyId); rs = pstmt.executeQuery(); if (rs.next()) { CopyNumberSegmentFile cnsf = new CopyNumberSegmentFile(); - cnsf.segFileId = rs.getInt("SEG_FILE_ID"); + cnsf.segFileId = rs.getInt("seg_file_id"); cnsf.cancerStudyId = cancerStudyId; - cnsf.referenceGenomeId = CopyNumberSegmentFile.ReferenceGenomeId.valueOf(rs.getString("REFERENCE_GENOME_ID")); - cnsf.description = rs.getString("DESCRIPTION"); - cnsf.filename = rs.getString("FILENAME"); + cnsf.referenceGenomeId = CopyNumberSegmentFile.ReferenceGenomeId.valueOf(rs.getString("reference_genome_id")); + cnsf.description = rs.getString("description"); + cnsf.filename = rs.getString("filename"); if (rs.next()) { throw new SQLException("More than one row was returned."); } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGene.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGene.java index b5c0352b..90efc36c 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGene.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGene.java @@ -86,13 +86,13 @@ public static int updateGene(CanonicalGene gene) throws DaoException { //when it is called in a process that may update a gene more than once //(e.g. the ImportGeneData updates some of the fields based on one // input file and other fields based on another input file): - setBulkLoadAtEnd = MySQLbulkLoader.isBulkLoad(); - MySQLbulkLoader.bulkLoadOff(); + setBulkLoadAtEnd = ClickHouseBulkLoader.isBulkLoad(); + ClickHouseBulkLoader.bulkLoadOff(); int rows = 0; con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("UPDATE gene SET `HUGO_GENE_SYMBOL`=?, `TYPE`=? WHERE `ENTREZ_GENE_ID`=?"); + ("UPDATE gene SET `hugo_gene_symbol`=?, `type`=? WHERE `entrez_gene_id`=?"); pstmt.setString(1, gene.getHugoGeneSymbolAllCaps()); pstmt.setString(2, gene.getType()); pstmt.setLong(3, gene.getEntrezGeneId()); @@ -107,7 +107,7 @@ public static int updateGene(CanonicalGene gene) throws DaoException { } finally { if (setBulkLoadAtEnd) { //reset to original state: - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } JdbcUtil.closeAll(DaoGene.class, con, pstmt, rs); } @@ -144,7 +144,7 @@ public static int addOrUpdateGene(CanonicalGene gene) throws DaoException { //add gene, referring to this genetic entity con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("INSERT INTO gene (`GENETIC_ENTITY_ID`, `ENTREZ_GENE_ID`,`HUGO_GENE_SYMBOL`,`TYPE`) " + ("INSERT INTO gene (`genetic_entity_id`, `entrez_gene_id`,`hugo_gene_symbol`,`type`) " + "VALUES (?,?,?,?)"); pstmt.setInt(1, geneticEntityId); pstmt.setLong(2, gene.getEntrezGeneId()); @@ -180,11 +180,11 @@ public static int addOrUpdateGene(CanonicalGene gene) throws DaoException { * @throws DaoException Database Error. */ public static int addGeneAliases(CanonicalGene gene) throws DaoException { - if (MySQLbulkLoader.isBulkLoad()) { - // write to the temp file maintained by the MySQLbulkLoader + if (ClickHouseBulkLoader.isBulkLoad()) { + // write to the temp file maintained by the ClickHouseBulkLoader Set aliases = gene.getAliases(); for (String alias : aliases) { - MySQLbulkLoader.getMySQLbulkLoader("gene_alias").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("gene_alias").insertRecord( Long.toString(gene.getEntrezGeneId()), alias); @@ -204,7 +204,7 @@ public static int addGeneAliases(CanonicalGene gene) throws DaoException { for (String alias : aliases) { if (!existingAliases.contains(alias)) { pstmt = con.prepareStatement("INSERT INTO gene_alias " - + "(`ENTREZ_GENE_ID`,`GENE_ALIAS`) VALUES (?,?)"); + + "(`entrez_gene_id`,`gene_alias`) VALUES (?,?)"); pstmt.setLong(1, gene.getEntrezGeneId()); pstmt.setString(2, alias); rows += pstmt.executeUpdate(); @@ -234,7 +234,7 @@ private static CanonicalGene getGene(long entrezGeneId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("SELECT * FROM gene WHERE ENTREZ_GENE_ID = ?"); + ("SELECT * FROM gene WHERE entrez_gene_id = ?"); pstmt.setLong(1, entrezGeneId); rs = pstmt.executeQuery(); if (rs.next()) { @@ -261,12 +261,12 @@ private static Set getAliases(long entrezGeneId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("SELECT * FROM gene_alias WHERE ENTREZ_GENE_ID = ?"); + ("SELECT * FROM gene_alias WHERE entrez_gene_id = ?"); pstmt.setLong(1, entrezGeneId); rs = pstmt.executeQuery(); Set aliases = new HashSet(); while (rs.next()) { - aliases.add(rs.getString("GENE_ALIAS")); + aliases.add(rs.getString("gene_alias")); } return aliases; } catch (SQLException e) { @@ -287,13 +287,13 @@ private static Map> getAllAliases() throws DaoException { rs = pstmt.executeQuery(); Map> map = new HashMap>(); while (rs.next()) { - Long entrez = rs.getLong("ENTREZ_GENE_ID"); + Long entrez = rs.getLong("entrez_gene_id"); Set aliases = map.get(entrez); if (aliases==null) { aliases = new HashSet(); map.put(entrez, aliases); } - aliases.add(rs.getString("GENE_ALIAS")); + aliases.add(rs.getString("gene_alias")); } return map; } catch (SQLException e) { @@ -321,12 +321,12 @@ public static ArrayList getAllGenes() throws DaoException { ("SELECT * FROM gene"); rs = pstmt.executeQuery(); while (rs.next()) { - int geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - long entrezGeneId = rs.getInt("ENTREZ_GENE_ID"); + int geneticEntityId = rs.getInt("genetic_entity_id"); + long entrezGeneId = rs.getInt("entrez_gene_id"); Set aliases = mapAliases.get(entrezGeneId); CanonicalGene gene = new CanonicalGene(geneticEntityId, entrezGeneId, - rs.getString("HUGO_GENE_SYMBOL"), aliases); - gene.setType(rs.getString("TYPE")); + rs.getString("hugo_gene_symbol"), aliases); + gene.setType(rs.getString("type")); geneList.add(gene); } return geneList; @@ -352,7 +352,7 @@ private static CanonicalGene getGene(String hugoGeneSymbol) throws DaoException try { con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("SELECT * FROM gene WHERE HUGO_GENE_SYMBOL = ?"); + ("SELECT * FROM gene WHERE hugo_gene_symbol = ?"); pstmt.setString(1, hugoGeneSymbol); rs = pstmt.executeQuery(); if (rs.next()) { @@ -368,12 +368,12 @@ private static CanonicalGene getGene(String hugoGeneSymbol) throws DaoException } private static CanonicalGene extractGene(ResultSet rs) throws SQLException, DaoException { - int geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - long entrezGeneId = rs.getInt("ENTREZ_GENE_ID"); + int geneticEntityId = rs.getInt("genetic_entity_id"); + long entrezGeneId = rs.getInt("entrez_gene_id"); Set aliases = getAliases(entrezGeneId); CanonicalGene gene = new CanonicalGene(geneticEntityId, entrezGeneId, - rs.getString("HUGO_GENE_SYMBOL"), aliases); - gene.setType(rs.getString("TYPE")); + rs.getString("hugo_gene_symbol"), aliases); + gene.setType(rs.getString("type")); return gene; } @@ -391,7 +391,7 @@ public static int getCount() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("SELECT COUNT(*) FROM gene"); + ("SELECT count(*) FROM gene"); rs = pstmt.executeQuery(); if (rs.next()) { return rs.getInt(1); @@ -417,7 +417,7 @@ public static void deleteGene(long entrezGeneId) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGene.class); - pstmt = con.prepareStatement("DELETE FROM gene WHERE ENTREZ_GENE_ID=?"); + pstmt = con.prepareStatement("DELETE FROM gene WHERE entrez_gene_id=?"); pstmt.setLong(1, entrezGeneId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -438,7 +438,7 @@ public static void deleteGeneAlias(long entrezGeneId) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGene.class); - pstmt = con.prepareStatement("DELETE FROM gene_alias WHERE ENTREZ_GENE_ID=?"); + pstmt = con.prepareStatement("DELETE FROM gene_alias WHERE entrez_gene_id=?"); pstmt.setLong(1, entrezGeneId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -461,10 +461,8 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGene.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE gene"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenePanel.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenePanel.java index b7efd650..0139de54 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenePanel.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenePanel.java @@ -40,10 +40,14 @@ import org.mskcc.cbio.portal.util.GenePanelUtil; import org.mskcc.cbio.portal.util.GenePanelUtil.Pair; +import static org.mskcc.cbio.portal.util.GenePanelUtil.getAddRemove; + /** * @author heinsz */ public class DaoGenePanel { + + private static final String GENE_PANEL_SEQUENCE = "seq_gene_panel"; private static final Map genePanelMap = initMap(); /** @@ -84,9 +88,9 @@ private static Map extractGenePanelMap(ResultSet rs) throws D try { while (rs.next()) { GenePanel gp = new GenePanel(); - gp.setInternalId(rs.getInt("INTERNAL_ID")); - gp.setStableId(rs.getString("STABLE_ID")); - gp.setDescription(rs.getString("DESCRIPTION")); + gp.setInternalId(rs.getInt("internal_id")); + gp.setStableId(rs.getString("stable_id")); + gp.setDescription(rs.getString("description")); gp.setGenes(extractGenePanelGenes(gp.getInternalId())); genePanelMap.put(gp.getStableId(), gp); } @@ -103,7 +107,7 @@ private static Set extractGenePanelGenes(Integer genePanelId) thr HashSet toReturn = new HashSet<>(); try { con = JdbcUtil.getDbConnection(DaoGenePanel.class); - pstmt = con.prepareStatement("SELECT * FROM gene_panel_list where INTERNAL_ID = ?"); + pstmt = con.prepareStatement("SELECT * FROM gene_panel_list where internal_id = ?"); pstmt.setInt(1, genePanelId); rs = pstmt.executeQuery(); DaoGeneOptimized daoGeneOpt = DaoGeneOptimized.getInstance(); @@ -142,26 +146,23 @@ public static void addGenePanel(String stableId, String description, Set try { con = JdbcUtil.getDbConnection(DaoGenePanel.class); for (CanonicalGene canonicalGene : canonicalGenes) { - pstmt = con.prepareStatement("INSERT INTO gene_panel_list (`INTERNAL_ID`, `GENE_ID`) VALUES (?,?)"); + pstmt = con.prepareStatement("INSERT INTO gene_panel_list (`internal_id`, `gene_id`) VALUES (?,?)"); pstmt.setInt(1, internalId); pstmt.setLong(2, canonicalGene.getEntrezGeneId()); pstmt.executeUpdate(); @@ -195,7 +196,7 @@ public static void deleteGenePanel(GenePanel genePanel) throws DaoException { PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoGenePanel.class); - pstmt = con.prepareStatement("DELETE from gene_panel WHERE INTERNAL_ID = ?"); + pstmt = con.prepareStatement("DELETE from gene_panel WHERE internal_id = ?"); pstmt.setInt(1, genePanel.getInternalId()); pstmt.executeUpdate(); genePanelMap.remove(genePanel.getStableId()); @@ -232,7 +233,7 @@ public static void updateGenePanel(GenePanel genePanel, Set incom * @throws DaoException */ public static void updatePreview(GenePanel genePanel, Set incoming) throws DaoException { - Pair pair = GenePanelUtil.getAddRemove(incoming, extractGenePanelGenes(genePanel.getInternalId())); + Pair pair = getAddRemove(incoming, extractGenePanelGenes(genePanel.getInternalId())); String add = pair.add.stream() .map(CanonicalGene::toString) .collect(Collectors.joining(", ")); @@ -280,13 +281,13 @@ private static void updateGenePanelGeneList(Integer internalId, Set toAdd = pair.add; Set toRemove = pair.remove; // Add and remove genes from specified gene panel for (CanonicalGene canonicalGene : toAdd) { - pstmt = con.prepareStatement("INSERT INTO gene_panel_list (`INTERNAL_ID`, `GENE_ID`) VALUES (?,?)"); + pstmt = con.prepareStatement("INSERT INTO gene_panel_list (`internal_id`, `gene_id`) VALUES (?,?)"); pstmt.setInt(1, internalId); pstmt.setLong(2, canonicalGene.getEntrezGeneId()); pstmt.executeUpdate(); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenericAssay.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenericAssay.java index c14efa3f..a14aa14c 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenericAssay.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenericAssay.java @@ -19,7 +19,7 @@ public static void setGenericEntityProperty(Integer entityId, String name, Strin try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - pstmt = con.prepareStatement("INSERT INTO generic_entity_properties (`GENETIC_ENTITY_ID`, `NAME`, `VALUE`) " + pstmt = con.prepareStatement("INSERT INTO generic_entity_properties (`genetic_entity_id`, `name`, `value`) " + "VALUES(?,?,?)"); if (entityId == null) { return; @@ -43,7 +43,7 @@ public static void setGenericEntityPropertiesUsingBatch(List map = new HashMap<>(); while(rs.next()) { - map.put(rs.getString("NAME"), rs.getString("VALUE")); + map.put(rs.getString("name"), rs.getString("value")); } GenericAssayMeta genericAssayMeta = new GenericAssayMeta(entity.getEntityType(), entity.getStableId(), map); return genericAssayMeta; @@ -108,7 +108,7 @@ public static void deleteGenericEntityPropertiesByEntityId(Integer entityId) thr try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - pstmt = con.prepareStatement("DELETE FROM generic_entity_properties WHERE GENETIC_ENTITY_ID=?"); + pstmt = con.prepareStatement("DELETE FROM generic_entity_properties WHERE genetic_entity_id=?"); if (entityId == null) { return; } @@ -128,13 +128,13 @@ public static boolean geneticEntitiesOnlyExistInSingleStudy(int geneticProfileId try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - pstmt = con.prepareStatement("SELECT DISTINCT CANCER_STUDY_ID FROM genetic_profile WHERE GENETIC_PROFILE_ID IN (SELECT GENETIC_PROFILE_ID FROM genetic_alteration WHERE GENETIC_ENTITY_ID IN (SELECT GENETIC_ENTITY_ID FROM genetic_alteration WHERE GENETIC_PROFILE_ID=?))"); + pstmt = con.prepareStatement("SELECT DISTINCT cancer_study_id FROM genetic_profile WHERE genetic_profile_id IN (SELECT genetic_profile_id FROM genetic_alteration WHERE genetic_entity_id IN (SELECT genetic_entity_id FROM genetic_alteration WHERE genetic_profile_id=?))"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); List studies = new ArrayList(); while(rs.next()) { - studies.add(rs.getInt("CANCER_STUDY_ID")); + studies.add(rs.getInt("cancer_study_id")); } // check if entities only exist in single study return studies.size() == 1; @@ -159,8 +159,8 @@ public static void checkAndDeleteGenericAssayMetaInStudy(int internalCancerStudy for (GeneticProfile profile : genericAssayProfiles) { if (DaoGenericAssay.geneticEntitiesOnlyExistInSingleStudy(profile.getGeneticProfileId())) { - deleteGenericAssayStatements.add(Pair.of(profile.getGeneticProfileId(), "DELETE FROM generic_entity_properties WHERE GENETIC_ENTITY_ID IN (SELECT GENETIC_ENTITY_ID FROM genetic_alteration WHERE GENETIC_PROFILE_ID=?)")); - deleteGenericAssayStatements.add(Pair.of(profile.getGeneticProfileId(), "DELETE FROM genetic_entity WHERE ID IN (SELECT GENETIC_ENTITY_ID FROM genetic_alteration WHERE GENETIC_PROFILE_ID=?)")); + deleteGenericAssayStatements.add(Pair.of(profile.getGeneticProfileId(), "DELETE FROM generic_entity_properties WHERE genetic_entity_id IN (SELECT genetic_entity_id FROM genetic_alteration WHERE genetic_profile_id=?)")); + deleteGenericAssayStatements.add(Pair.of(profile.getGeneticProfileId(), "DELETE FROM genetic_entity WHERE id IN (SELECT genetic_entity_id FROM genetic_alteration WHERE genetic_profile_id=?)")); } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneset.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneset.java index 65f5fee1..4f186ec0 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneset.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneset.java @@ -50,6 +50,8 @@ public class DaoGeneset { + private static final String GENESET_SEQUENCE = "seq_geneset"; + private DaoGeneset() { } @@ -70,20 +72,18 @@ public static Geneset addGeneset(Geneset geneset) throws DaoException { geneset.setGeneticEntityId(geneticEntityId); con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("INSERT INTO geneset " - + "(`GENETIC_ENTITY_ID`, `EXTERNAL_ID`, `NAME`, `DESCRIPTION`, `REF_LINK`) " - + "VALUES(?,?,?,?,?)", Statement.RETURN_GENERATED_KEYS); - pstmt.setInt(1, geneset.getGeneticEntityId()); - pstmt.setString(2, geneset.getExternalId()); - pstmt.setString(3, geneset.getName()); - pstmt.setString(4, geneset.getDescription()); - pstmt.setString(5, geneset.getRefLink()); + long genesetId = ClickHouseAutoIncrement.nextId(GENESET_SEQUENCE); + pstmt = con.prepareStatement("INSERT INTO geneset " + + "(`id`, `genetic_entity_id`, `external_id`, `name`, `description`, `ref_link`) " + + "VALUES(?,?,?,?,?,?)"); + pstmt.setLong(1, genesetId); + pstmt.setInt(2, geneset.getGeneticEntityId()); + pstmt.setString(3, geneset.getExternalId()); + pstmt.setString(4, geneset.getName()); + pstmt.setString(5, geneset.getDescription()); + pstmt.setString(6, geneset.getRefLink()); pstmt.executeUpdate(); - //get the auto generated key: - rs = pstmt.getGeneratedKeys(); - rs.next(); - int newId = rs.getInt(1); - geneset.setId(newId); + geneset.setId((int) genesetId); return geneset; } @@ -96,7 +96,7 @@ public static Geneset addGeneset(Geneset geneset) throws DaoException { } /** - * Prepares a list of Gene records from Geneset object to be added to database via MySQLbulkLoader. + * Prepares a list of Gene records from Geneset object to be added to database via ClickHouseBulkLoader. * @param geneset * @return number of records where entrez gene id is found in db */ @@ -114,8 +114,8 @@ public static int addGenesetGenesToBulkLoader(Geneset geneset) { continue; } // use this code if bulk loading - // write to the temp file maintained by the MySQLbulkLoader - MySQLbulkLoader.getMySQLbulkLoader("geneset_gene").insertRecord( + // write to the temp file maintained by the ClickHouseBulkLoader + ClickHouseBulkLoader.getClickHouseBulkLoader("geneset_gene").insertRecord( Integer.toString(geneset.getId()), Long.toString(entrezGeneId)); rows ++; @@ -137,14 +137,14 @@ public static List getGenesetGenes(Geneset geneset) throws DaoExc try { con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("SELECT * FROM geneset_gene WHERE GENESET_ID = ?"); + pstmt = con.prepareStatement("SELECT * FROM geneset_gene WHERE geneset_id = ?"); pstmt.setInt(1, geneset.getId()); rs = pstmt.executeQuery(); // get list of entrez gene ids for geneset record Set entrezGeneIds = new HashSet(); while (rs.next()) { - entrezGeneIds.add(rs.getLong("ENTREZ_GENE_ID")); + entrezGeneIds.add(rs.getLong("entrez_gene_id")); } // get list of genes by entrez gene ids @@ -176,7 +176,7 @@ public static Geneset getGenesetByExternalId(String externalId) throws DaoExcept ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("SELECT * FROM geneset WHERE `EXTERNAL_ID` = ?"); + pstmt = con.prepareStatement("SELECT * FROM geneset WHERE `external_id` = ?"); pstmt.setString(1, externalId); rs = pstmt.executeQuery(); @@ -227,12 +227,12 @@ public static List getAllGenesets() throws DaoException { * @throws DaoException */ private static Geneset extractGeneset(ResultSet rs) throws SQLException, DaoException { - Integer id = rs.getInt("ID"); - Integer geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - String externalId = rs.getString("EXTERNAL_ID"); - String name = rs.getString("NAME"); - String description = rs.getString("DESCRIPTION"); - String refLink = rs.getString("REF_LINK"); + Integer id = rs.getInt("id"); + Integer geneticEntityId = rs.getInt("genetic_entity_id"); + String externalId = rs.getString("external_id"); + String name = rs.getString("name"); + String description = rs.getString("description"); + String refLink = rs.getString("ref_link"); Geneset geneset = new Geneset(); geneset.setId(id); @@ -254,12 +254,12 @@ public static Set getGenesetGeneticEntityIds() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("SELECT ID FROM genetic_entity WHERE ENTITY_TYPE = 'GENESET'"); + pstmt = con.prepareStatement("SELECT id FROM genetic_entity WHERE entity_type = 'geneset'"); rs = pstmt.executeQuery(); Set geneticEntities = new HashSet(); while (rs.next()) { - geneticEntities.add(rs.getLong("ID")); + geneticEntities.add(rs.getLong("id")); } return geneticEntities; @@ -277,8 +277,8 @@ public static Set getGenesetGeneticEntityIds() throws DaoException { * @throws DaoException */ public static boolean checkUsage(Integer geneticEntityId) throws DaoException { - String SQL = "SELECT COUNT(DISTINCT `CANCER_STUDY_ID`) FROM genetic_profile " + - "WHERE `GENETIC_PROFILE_ID` IN (SELECT `GENETIC_PROFILE_ID` FROM genetic_alteration WHERE `GENETIC_ENTITY_ID` = ?)"; + String SQL = "SELECT count(DISTINCT `cancer_study_id`) FROM genetic_profile " + + "WHERE `genetic_profile_id` IN (SELECT `genetic_profile_id` FROM genetic_alteration WHERE `genetic_entity_id` = ?)"; Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; @@ -302,8 +302,8 @@ public static boolean checkUsage(Integer geneticEntityId) throws DaoException { public static void updateGeneset(Geneset geneset, boolean updateGenesetGenes) throws DaoException { String SQL = "UPDATE geneset SET " + - "`NAME` = ?, `DESCRIPTION` = ?, `REF_LINK` = ?" + - "WHERE `ID` = ?"; + "`name` = ?, `description` = ?, `ref_link` = ?" + + "WHERE `id` = ?"; Connection con = null; PreparedStatement pstmt = null; @@ -338,7 +338,7 @@ private static void deleteGenesetGeneticEntityRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("DELETE FROM genetic_entity WHERE ENTITY_TYPE = 'GENESET'"); + pstmt = con.prepareStatement("DELETE FROM genetic_entity WHERE entity_type = 'geneset'"); pstmt.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); @@ -360,7 +360,7 @@ private static void deleteGenesetGeneticProfiles() throws DaoException { connection = JdbcUtil.getDbConnection(DaoGeneset.class); // Prepare statement - preparedStatement = connection.prepareStatement("DELETE FROM genetic_profile WHERE GENETIC_ALTERATION_TYPE = 'GENESET_SCORE'"); + preparedStatement = connection.prepareStatement("DELETE FROM genetic_profile WHERE genetic_alteration_type = 'geneset_score'"); // Execute statement preparedStatement.executeUpdate(); @@ -379,17 +379,28 @@ private static void deleteGenesetGeneticProfiles() throws DaoException { */ private static void deleteGenesetGeneticProfileLinks() throws DaoException { Connection connection = null; - PreparedStatement preparedStatement = null; ResultSet resultSet = null; + PreparedStatement preparedStatement = null; try { connection = JdbcUtil.getDbConnection(DaoGeneset.class); - - // Prepare statement: DELETE genetic_profile_link which are pointing to a profile of type to GENESET_SCORE - preparedStatement = connection.prepareStatement("DELETE FROM genetic_profile_link WHERE REFERRED_GENETIC_PROFILE_ID IN " - + "(SELECT GENETIC_PROFILE_ID FROM genetic_profile WHERE GENETIC_ALTERATION_TYPE = 'GENESET_SCORE')"); - - // Execute statement - preparedStatement.executeUpdate(); + List genesetProfiles = collectGenesetProfileIds(connection); + if (genesetProfiles.isEmpty()) { + return; + } + final int batchSize = 500; + for (int start = 0; start < genesetProfiles.size(); start += batchSize) { + int end = Math.min(start + batchSize, genesetProfiles.size()); + List chunk = genesetProfiles.subList(start, end); + String placeholders = String.join(",", Collections.nCopies(chunk.size(), "?")); + JdbcUtil.closeAll(DaoGeneset.class, null, preparedStatement, null); + preparedStatement = connection.prepareStatement( + "DELETE FROM genetic_profile_link WHERE referred_genetic_profile_id IN (" + placeholders + ")"); + int parameterIndex = 1; + for (Integer id : chunk) { + preparedStatement.setInt(parameterIndex++, id); + } + preparedStatement.executeUpdate(); + } } catch (SQLException e) { throw new DaoException(e); @@ -397,7 +408,24 @@ private static void deleteGenesetGeneticProfileLinks() throws DaoException { finally { JdbcUtil.closeAll(DaoGeneset.class, connection, preparedStatement, resultSet); } - } + } + + private static List collectGenesetProfileIds(Connection connection) throws SQLException { + List ids = new ArrayList<>(); + PreparedStatement preparedStatement = null; + ResultSet rs = null; + try { + preparedStatement = connection.prepareStatement( + "SELECT genetic_profile_id FROM genetic_profile WHERE genetic_alteration_type = 'GENESET_SCORE'"); + rs = preparedStatement.executeQuery(); + while (rs.next()) { + ids.add(rs.getInt(1)); + } + } finally { + JdbcUtil.closeAll(DaoGeneset.class, null, preparedStatement, rs); + } + return ids; + } /** * Deletes all records from 'geneset' table in database and records in related tables. @@ -410,7 +438,7 @@ private static void deleteAllGenesetRecords() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGeneset.class); - pstmt = con.prepareStatement("DELETE FROM geneset"); + pstmt = con.prepareStatement("TRUNCATE TABLE geneset"); pstmt.executeUpdate(); } catch (SQLException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyLeaf.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyLeaf.java index 9b3fe2c1..37bb50de 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyLeaf.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyLeaf.java @@ -51,7 +51,7 @@ public static void addGenesetHierarchyLeaf(GenesetHierarchyLeaf genesetHierarchy // Prepare SQL statement preparedStatement = connection.prepareStatement("INSERT INTO geneset_hierarchy_leaf " - + "(`NODE_ID`, `GENESET_ID`) VALUES(?,?)"); + + "(`node_id`, `geneset_id`) VALUES(?,?)"); // Fill in statement preparedStatement.setInt(1, genesetHierarchyLeaf.getNodeId()); @@ -77,7 +77,7 @@ public static List getGenesetHierarchyLeafsByGenesetId(int connection = JdbcUtil.getDbConnection(DaoGenesetHierarchyLeaf.class); // Prepare SQL statement - preparedStatement = connection.prepareStatement("SELECT * FROM geneset_hierarchy_leaf WHERE GENESET_ID = ?"); + preparedStatement = connection.prepareStatement("SELECT * FROM geneset_hierarchy_leaf WHERE geneset_id = ?"); preparedStatement.setInt(1, genesetId); // Execute statement @@ -87,8 +87,8 @@ public static List getGenesetHierarchyLeafsByGenesetId(int while (resultSet.next()) { GenesetHierarchyLeaf genesetHierarchyLeaf = new GenesetHierarchyLeaf(); - genesetHierarchyLeaf.setNodeId(resultSet.getInt("NODE_ID")); - genesetHierarchyLeaf.setGenesetId(resultSet.getInt("GENESET_ID")); + genesetHierarchyLeaf.setNodeId(resultSet.getInt("node_id")); + genesetHierarchyLeaf.setGenesetId(resultSet.getInt("geneset_id")); genesetHierarchyLeafs.add(genesetHierarchyLeaf); } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyNode.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyNode.java index e13b5838..ac923436 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyNode.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGenesetHierarchyNode.java @@ -31,7 +31,9 @@ import org.mskcc.cbio.portal.model.GenesetHierarchy; public class DaoGenesetHierarchyNode { - + + private static final String GENESET_HIERARCHY_SEQUENCE = "seq_geneset_hierarchy_node"; + private DaoGenesetHierarchyNode() { } @@ -42,7 +44,6 @@ private DaoGenesetHierarchyNode() { public static void addGenesetHierarchy(GenesetHierarchy genesetHierarchy) throws DaoException { Connection connection = null; PreparedStatement preparedStatement = null; - ResultSet resultSet = null; try { // Open connection to database @@ -50,29 +51,28 @@ public static void addGenesetHierarchy(GenesetHierarchy genesetHierarchy) throws // Prepare SQL statement preparedStatement = connection.prepareStatement("INSERT INTO geneset_hierarchy_node " - + "(`NODE_NAME`, `PARENT_ID`) VALUES(?,?)", Statement.RETURN_GENERATED_KEYS); + + "(`node_id`, `node_name`, `parent_id`) VALUES(?,?,?)"); + long nodeId = ClickHouseAutoIncrement.nextId(GENESET_HIERARCHY_SEQUENCE); + preparedStatement.setLong(1, nodeId); // Fill in statement - preparedStatement.setString(1, genesetHierarchy.getNodeName()); + preparedStatement.setString(2, genesetHierarchy.getNodeName()); if (genesetHierarchy.getParentId() == 0) { - preparedStatement.setNull(2, java.sql.Types.INTEGER); + preparedStatement.setNull(3, java.sql.Types.INTEGER); } else { - preparedStatement.setInt(2, genesetHierarchy.getParentId()); + preparedStatement.setInt(3, genesetHierarchy.getParentId()); } // Execute statement preparedStatement.executeUpdate(); // Get the auto generated key, which is the Node ID: - resultSet = preparedStatement.getGeneratedKeys(); - if (resultSet.next()) { - genesetHierarchy.setNodeId(resultSet.getInt(1)); - } + genesetHierarchy.setNodeId((int) nodeId); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoGenesetHierarchyNode.class, connection, preparedStatement, resultSet); + JdbcUtil.closeAll(DaoGenesetHierarchyNode.class, connection, preparedStatement, null); } } @@ -118,7 +118,7 @@ public static void deleteAllGenesetHierarchyRecords() throws DaoException { ResultSet resultSet = null; try { connection = JdbcUtil.getDbConnection(DaoGenesetHierarchyNode.class); - preparedStatement = connection.prepareStatement("DELETE FROM geneset_hierarchy_node"); + preparedStatement = connection.prepareStatement("TRUNCATE TABLE geneset_hierarchy_node"); preparedStatement.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticAlteration.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticAlteration.java index efc92ea1..48cb5144 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticAlteration.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticAlteration.java @@ -111,9 +111,9 @@ public int addGeneticAlterationsForGeneticEntity(int geneticProfileId, int genet valueBuffer.append(value).append(DELIM); } - if (MySQLbulkLoader.isBulkLoad() ) { - // write to the temp file maintained by the MySQLbulkLoader - MySQLbulkLoader.getMySQLbulkLoader("genetic_alteration").insertRecord(Integer.toString( geneticProfileId ), + if (ClickHouseBulkLoader.isBulkLoad() ) { + // write to the temp file maintained by the ClickHouseBulkLoader + ClickHouseBulkLoader.getClickHouseBulkLoader("genetic_alteration").insertRecord(Integer.toString( geneticProfileId ), Integer.toString( geneticEntityId ), valueBuffer.toString()); // return 1 because normal insert will return 1 if no error occurs return 1; @@ -125,10 +125,8 @@ public int addGeneticAlterationsForGeneticEntity(int geneticProfileId, int genet try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); - pstmt = con.prepareStatement - ("INSERT INTO genetic_alteration (GENETIC_PROFILE_ID, " + - " GENETIC_ENTITY_ID," + - " `VALUES`) " + pstmt = con.prepareStatement( + "INSERT INTO genetic_alteration (genetic_profile_id, genetic_entity_id, `values`) " + "VALUES (?,?,?)"); pstmt.setInt(1, geneticProfileId); pstmt.setLong(2, geneticEntityId); @@ -227,19 +225,23 @@ public HashMap> getGeneticAlterationMapForEntit } try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); - if (geneticEntityIds == null) { - pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE" - + " GENETIC_PROFILE_ID = " + geneticProfileId); + if (geneticEntityIds == null || geneticEntityIds.isEmpty()) { + pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE genetic_profile_id = ?"); + pstmt.setInt(1, geneticProfileId); } else { - pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE" - + " GENETIC_PROFILE_ID = " + geneticProfileId - + " AND GENETIC_ENTITY_ID IN ("+StringUtils.join(geneticEntityIds, ",")+")"); + String placeholders = String.join(",", Collections.nCopies(geneticEntityIds.size(), "?")); + pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE genetic_profile_id = ? AND genetic_entity_id IN (" + placeholders + ")"); + pstmt.setInt(1, geneticProfileId); + int index = 2; + for (Integer geneticEntityId : geneticEntityIds) { + pstmt.setInt(index++, geneticEntityId); + } } rs = pstmt.executeQuery(); while (rs.next()) { HashMap mapSampleValue = new HashMap(); - int geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - String values = rs.getString("VALUES"); + int geneticEntityId = rs.getInt("genetic_entity_id"); + String values = rs.getString("values"); String[] valueParts = values.split(DELIM, -1); int valuesLength = valueParts.length; boolean hasMeaninglessTrailingDelimiter = valuesLength - orderedSampleList.size() == 1 && valueParts[valuesLength - 1].isEmpty(); @@ -250,7 +252,7 @@ public HashMap> getGeneticAlterationMapForEntit if (valuesLength != orderedSampleList.size()) { throw new IllegalStateException( "Data inconsistency detected: The length of the values for genetic profile with Id = " - + geneticProfileId + " and genetic entity with ID = " + geneticEntityId + + geneticProfileId + " and genetic entity with id = " + geneticEntityId + " (" + valuesLength + " elements) does not match the expected length of the sample list (" + orderedSampleList.size() + " elements)."); } @@ -299,14 +301,12 @@ public static ArrayList getProcessedAlterationData( try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); - pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE" - + " GENETIC_PROFILE_ID = " + geneticProfileId - + " LIMIT 3000 OFFSET " + offSet); - + pstmt = con.prepareStatement("SELECT * FROM genetic_alteration WHERE genetic_profile_id = ? LIMIT 3000 OFFSET " + offSet); + pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { - long entrezGeneId = DaoGeneOptimized.getEntrezGeneId(rs.getInt("GENETIC_ENTITY_ID")); - String valuesString = rs.getString("VALUES"); + long entrezGeneId = DaoGeneOptimized.getEntrezGeneId(rs.getInt("genetic_entity_id")); + String valuesString = rs.getString("values"); if (valuesString.endsWith(DELIM)) { valuesString = valuesString.substring(0, valuesString.length() - DELIM.length()); } @@ -342,12 +342,12 @@ public Set getGenesInProfile(int geneticProfileId) throws DaoExce try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); pstmt = con.prepareStatement - ("SELECT * FROM genetic_alteration WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT * FROM genetic_alteration WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { - Long entrezGeneId = DaoGeneOptimized.getEntrezGeneId(rs.getInt("GENETIC_ENTITY_ID")); + Long entrezGeneId = DaoGeneOptimized.getEntrezGeneId(rs.getInt("genetic_entity_id")); geneList.add(daoGene.getGene(entrezGeneId)); } return geneList; @@ -373,12 +373,12 @@ public static Set getEntityIdsInProfile(int geneticProfileId) throws Da try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); pstmt = con.prepareStatement - ("SELECT * FROM genetic_alteration WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT * FROM genetic_alteration WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { - int geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); + int geneticEntityId = rs.getInt("genetic_entity_id"); geneticEntityList.add(geneticEntityId); } return geneticEntityList; @@ -402,7 +402,7 @@ public static int getGenesCountInProfile(int geneticProfileId) throws DaoExcepti try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); pstmt = con.prepareStatement - ("SELECT COUNT(*) FROM genetic_alteration WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT count(*) FROM genetic_alteration WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); if (rs.next()) { @@ -428,7 +428,7 @@ public int getCount() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); pstmt = con.prepareStatement - ("SELECT COUNT(*) FROM genetic_alteration"); + ("SELECT count(*) FROM genetic_alteration"); rs = pstmt.executeQuery(); if (rs.next()) { return rs.getInt(1); @@ -453,8 +453,7 @@ public void deleteAllRecordsInGeneticProfile(long geneticProfileId) throws DaoEx ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticAlteration.class); - pstmt = con.prepareStatement("DELETE from " + - "genetic_alteration WHERE GENETIC_PROFILE_ID=?"); + pstmt = con.prepareStatement("DELETE from genetic_alteration WHERE genetic_profile_id=?"); pstmt.setLong(1, geneticProfileId); pstmt.executeUpdate(); } catch (SQLException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java index d614bbc1..8f2ed303 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java @@ -17,7 +17,7 @@ private DaoGeneticEntity() { } private enum SqlAction { - INSERT, UPDATE, SELECT, DELETE + UPDATE, SELECT, DELETE } /** @@ -30,15 +30,23 @@ private enum SqlAction { public static GeneticEntity addNewGeneticEntity(GeneticEntity geneticEntity) throws DaoException { - DbContainer container = executeSQLstatment( - SqlAction.INSERT, - "INSERT INTO genetic_entity (`ENTITY_TYPE`, `STABLE_ID`) " - + "VALUES(?,?)", - geneticEntity.getEntityType(), - geneticEntity.getStableId() - ); - - geneticEntity.setId(container.getId()); + Connection con = null; + PreparedStatement pstmt = null; + try { + con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); + long entityId = ClickHouseAutoIncrement.nextId("seq_genetic_entity"); + pstmt = con.prepareStatement("INSERT INTO genetic_entity (`id`, `entity_type`, `stable_id`) " + + "VALUES(?,?,?)"); + pstmt.setLong(1, entityId); + pstmt.setString(2, geneticEntity.getEntityType()); + pstmt.setString(3, geneticEntity.getStableId()); + pstmt.executeUpdate(); + geneticEntity.setId((int) entityId); + } catch (SQLException e) { + throw new DaoException(e); + } finally { + JdbcUtil.closeAll(DaoGeneticEntity.class, con, pstmt, null); + } return geneticEntity; } @@ -50,7 +58,7 @@ public static GeneticEntity addNewGeneticEntity(GeneticEntity geneticEntity) thr * @throws DaoException */ public static GeneticEntity getGeneticEntityByStableId(String stableId) throws DaoException { - DbContainer container = executeSQLstatment(SqlAction.SELECT, "SELECT * FROM genetic_entity WHERE `STABLE_ID` = ?", stableId); + DbContainer container = executeSQLstatment(SqlAction.SELECT, "SELECT * FROM genetic_entity WHERE `stable_id` = ?", stableId); return container.getGeneticEntity(); } @@ -59,7 +67,7 @@ public static GeneticEntity getGeneticEntityByStableId(String stableId) throws D * @param id genetic_entity id */ public static GeneticEntity getGeneticEntityById(int id) throws DaoException { - DbContainer container = executeSQLstatment(SqlAction.SELECT, "SELECT * FROM genetic_entity WHERE ID = ?", String.valueOf(id)); + DbContainer container = executeSQLstatment(SqlAction.SELECT, "SELECT * FROM genetic_entity WHERE id = ?", String.valueOf(id)); return container.getGeneticEntity(); } @@ -96,9 +104,9 @@ public static List getAllGeneticEntities() throws DaoException { */ private static GeneticEntity extractGeneticEntity(ResultSet rs) throws SQLException, DaoException { - Integer id = rs.getInt("ID"); - String stableId = rs.getString("STABLE_ID"); - String entityType = rs.getString("ENTITY_TYPE"); + Integer id = rs.getInt("id"); + String stableId = rs.getString("stable_id"); + String entityType = rs.getString("entity_type"); GeneticEntity geneticEntity = new GeneticEntity(id, entityType, stableId); @@ -107,8 +115,8 @@ private static GeneticEntity extractGeneticEntity(ResultSet rs) throws SQLExcept /** * Helper method for retrieval of a geneticEntity record from the database - * @param action type of MySQL operation - * @param statement MySQL statement + * @param action type of SQL operation + * @param statement SQL statement * @param keys Series of values used in the statement (order is important) * @return Object return data from * @throws DaoException @@ -120,24 +128,13 @@ private static DbContainer executeSQLstatment(SqlAction action, String statement try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - // for insert statements the number of affected records is returned - // this requires the RETURN_GENERATED_KEYS to be set for insert statements. - int switchGetGeneratedKeys = PreparedStatement.NO_GENERATED_KEYS; - if (action == SqlAction.INSERT) - switchGetGeneratedKeys = PreparedStatement.RETURN_GENERATED_KEYS; - - pstmt = con.prepareStatement(statement, switchGetGeneratedKeys); + pstmt = con.prepareStatement(statement); int cnt = 1; for (int i = 0; i < keys.length; i++) pstmt.setString(cnt++, keys[i]); switch(action) { - case INSERT: - pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - rs.next(); - return new DbContainer(rs.getInt(1)); case SELECT: rs = pstmt.executeQuery(); if (rs.next()) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfile.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfile.java index dce69a95..f7fdc961 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfile.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfile.java @@ -42,6 +42,7 @@ import java.util.Map; import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; +import org.slf4j.Logger; /** * Analogous to and replaces the old DaoCancerType. A CancerStudy has a NAME and @@ -51,8 +52,9 @@ * Data access object for Genetic Profile table */ public final class DaoGeneticProfile { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(DaoGeneticProfile.class); private DaoGeneticProfile() {} - + private static final String GENETIC_PROFILE_SEQUENCE = "seq_genetic_profile"; private static final Map byStableId = new HashMap(); private static final Map byInternalId = new HashMap(); private static final Map> byStudy = new HashMap>(); @@ -70,6 +72,7 @@ public static synchronized void reCache() { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); + log.info("Connection obtained for recaching genetic_profile: " + con); pstmt = con.prepareStatement ("SELECT * FROM genetic_profile"); @@ -97,46 +100,54 @@ private static void cacheGeneticProfile(GeneticProfile profile) { } public static int addGeneticProfile(GeneticProfile profile) throws DaoException { + //FIXME: this cast may cause problems in the future + profile.setGeneticProfileId((int) ClickHouseAutoIncrement.nextId(GENETIC_PROFILE_SEQUENCE)); + log.info("Assigned genetic profile ID " + profile.getGeneticProfileId() + + " to stable ID " + profile.getStableId()); Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; int rows = 0; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); + log.info("Connection obtained for inserting genetic_profile: " + con); pstmt = con.prepareStatement - ("INSERT INTO genetic_profile (`STABLE_ID`, `CANCER_STUDY_ID`, "+ - "`GENETIC_ALTERATION_TYPE`, `DATATYPE`, `NAME`, `DESCRIPTION`, "+ - "`SHOW_PROFILE_IN_ANALYSIS_TAB`, `PIVOT_THRESHOLD`, `SORT_ORDER`, `GENERIC_ASSAY_TYPE`, `PATIENT_LEVEL`) " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?)"); - pstmt.setString(1, profile.getStableId()); - pstmt.setInt(2, profile.getCancerStudyId()); - pstmt.setString(3, profile.getGeneticAlterationType().name()); - pstmt.setString(4, profile.getDatatype()); - pstmt.setString(5, profile.getProfileName()); - pstmt.setString(6, profile.getProfileDescription()); - pstmt.setBoolean(7, profile.showProfileInAnalysisTab()); + ("INSERT INTO genetic_profile (`genetic_profile_id`, `stable_id`, `cancer_study_id`, "+ + "`genetic_alteration_type`, `datatype`, `name`, `description`, "+ + "`show_profile_in_analysis_tab`, `pivot_threshold`, `sort_order`, `generic_assay_type`, `patient_level`) " + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"); + + int index = 1; + pstmt.setInt(index++, profile.getGeneticProfileId()); + pstmt.setString(index++, profile.getStableId()); + pstmt.setInt(index++, profile.getCancerStudyId()); + pstmt.setString(index++, profile.getGeneticAlterationType().name()); + pstmt.setString(index++, profile.getDatatype()); + pstmt.setString(index++, profile.getProfileName()); + pstmt.setString(index++, profile.getProfileDescription()); + pstmt.setBoolean(index++, profile.showProfileInAnalysisTab()); - // `pivot_threshold_value` and `value_sort_order` `GENERIC_ASSAY_TYPE` fields are geneirc assay data specific. + // `pivot_threshold_value` and `value_sort_order` `generic_assay_type` fields are geneirc assay data specific. // These fields are set to null when not present in profile object. if (profile.getPivotThreshold() == null) { - pstmt.setNull(8, java.sql.Types.FLOAT); + pstmt.setNull(index++, java.sql.Types.FLOAT); } else { - pstmt.setFloat(8, profile.getPivotThreshold()); + pstmt.setFloat(index++, profile.getPivotThreshold()); } if (profile.getSortOrder() == null) { - pstmt.setNull(9, java.sql.Types.INTEGER); + pstmt.setNull(index++, java.sql.Types.INTEGER); } else { - pstmt.setString(9, profile.getSortOrder()); + pstmt.setString(index++, profile.getSortOrder()); } if (profile.getGenericAssayType() == null) { - pstmt.setNull(10, java.sql.Types.VARCHAR); + pstmt.setNull(index++, java.sql.Types.VARCHAR); } else { - pstmt.setString(10, profile.getGenericAssayType()); + pstmt.setString(index++, profile.getGenericAssayType()); } // default value is false - pstmt.setBoolean(11, profile.getPatientLevel()); + pstmt.setBoolean(index++, profile.getPatientLevel()); rows = pstmt.executeUpdate(); @@ -166,8 +177,8 @@ public static boolean updateNameAndDescription (int geneticProfileId, String nam boolean ret = false; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); - pstmt = con.prepareStatement("UPDATE genetic_profile SET NAME=?, DESCRIPTION=? " + - "WHERE GENETIC_PROFILE_ID=?"); + pstmt = con.prepareStatement("UPDATE genetic_profile SET name=?, description=? " + + "WHERE genetic_profile_id=?"); pstmt.setString(1, name); pstmt.setString(2, description); pstmt.setInt(3, geneticProfileId); @@ -195,8 +206,8 @@ public static boolean updateDatatype( boolean ret; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); - pstmt = con.prepareStatement("UPDATE genetic_profile SET DATATYPE=? " + - "WHERE GENETIC_PROFILE_ID=?"); + pstmt = con.prepareStatement("UPDATE genetic_profile SET datatype=? " + + "WHERE genetic_profile_id=?"); pstmt.setString(1, datatype); pstmt.setInt(2, geneticProfileId); ret = pstmt.executeUpdate() > 0; @@ -217,7 +228,7 @@ public static int deleteGeneticProfile(GeneticProfile profile) throws DaoExcepti ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); - pstmt = con.prepareStatement("DELETE FROM genetic_profile WHERE STABLE_ID = ?"); + pstmt = con.prepareStatement("DELETE FROM genetic_profile WHERE stable_id = ?"); pstmt.setString(1, profile.getStableId()); rows = pstmt.executeUpdate(); } catch (SQLException e) { @@ -261,29 +272,29 @@ public static int getCount() { private static GeneticProfile extractGeneticProfile(ResultSet rs) throws SQLException { GeneticProfile profileType = new GeneticProfile(); - profileType.setStableId(rs.getString("STABLE_ID")); + profileType.setStableId(rs.getString("stable_id")); - profileType.setCancerStudyId(rs.getInt("CANCER_STUDY_ID")); - profileType.setProfileName(rs.getString("NAME")); - profileType.setProfileDescription(rs.getString("DESCRIPTION")); + profileType.setCancerStudyId(rs.getInt("cancer_study_id")); + profileType.setProfileName(rs.getString("name")); + profileType.setProfileDescription(rs.getString("description")); try { - profileType.setShowProfileInAnalysisTab(rs.getBoolean("SHOW_PROFILE_IN_ANALYSIS_TAB")); + profileType.setShowProfileInAnalysisTab(rs.getBoolean("show_profile_in_analysis_tab")); } catch (SQLException e) { profileType.setShowProfileInAnalysisTab(true); } - profileType.setGeneticAlterationType(GeneticAlterationType.valueOf(rs.getString("GENETIC_ALTERATION_TYPE"))); - profileType.setDatatype(rs.getString("DATATYPE")); - profileType.setGeneticProfileId(rs.getInt("GENETIC_PROFILE_ID")); - if (rs.getFloat("PIVOT_THRESHOLD") != 0) { - profileType.setPivotThreshold(rs.getFloat("PIVOT_THRESHOLD")); + profileType.setGeneticAlterationType(GeneticAlterationType.valueOf(rs.getString("genetic_alteration_type"))); + profileType.setDatatype(rs.getString("datatype")); + profileType.setGeneticProfileId(rs.getInt("genetic_profile_id")); + if (rs.getFloat("pivot_threshold") != 0) { + profileType.setPivotThreshold(rs.getFloat("pivot_threshold")); } - if (rs.getString("SORT_ORDER") != null && ! rs.getString("SORT_ORDER").equals("") ) { - profileType.setSortOrder(rs.getString("SORT_ORDER")); + if (rs.getString("sort_order") != null && ! rs.getString("sort_order").equals("") ) { + profileType.setSortOrder(rs.getString("sort_order")); } - if (rs.getString("GENERIC_ASSAY_TYPE") != null && ! rs.getString("GENERIC_ASSAY_TYPE").equals("") ) { - profileType.setGenericAssayType(rs.getString("GENERIC_ASSAY_TYPE")); + if (rs.getString("generic_assay_type") != null && ! rs.getString("generic_assay_type").equals("") ) { + profileType.setGenericAssayType(rs.getString("generic_assay_type")); } - profileType.setPatientLevel(rs.getBoolean("PATIENT_LEVEL")); + profileType.setPatientLevel(rs.getBoolean("patient_level")); return profileType; } @@ -306,10 +317,8 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticProfile.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE genetic_profile"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileLink.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileLink.java index 8caddeb1..b0830ffd 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileLink.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileLink.java @@ -50,7 +50,7 @@ public static void addGeneticProfileLink(GeneticProfileLink geneticProfileLink) // Prepare SQL statement preparedStatement = connection.prepareStatement("INSERT INTO genetic_profile_link " - + "(REFERRING_GENETIC_PROFILE_ID, REFERRED_GENETIC_PROFILE_ID, REFERENCE_TYPE) VALUES(?,?,?)"); + + "(referring_genetic_profile_id, referred_genetic_profile_id, reference_type) VALUES(?,?,?)"); // Fill in statement preparedStatement.setInt(1, geneticProfileLink.getReferringGeneticProfileId()); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileSamples.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileSamples.java index a598f458..eeaf3565 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileSamples.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticProfileSamples.java @@ -67,8 +67,8 @@ public static int addGeneticProfileSamples(int geneticProfileId, List o try { con = JdbcUtil.getDbConnection(DaoGeneticProfileSamples.class); pstmt = con.prepareStatement - ("INSERT INTO genetic_profile_samples (`GENETIC_PROFILE_ID`, " + - "`ORDERED_SAMPLE_LIST`) "+ "VALUES (?,?)"); + ("INSERT INTO genetic_profile_samples (`genetic_profile_id`, " + + "`ordered_sample_list`) "+ "VALUES (?,?)"); pstmt.setInt(1, geneticProfileId); pstmt.setString(2, orderedSampleListBuf.toString()); return pstmt.executeUpdate(); @@ -93,8 +93,7 @@ public static void deleteAllSamplesInGeneticProfile(int geneticProfileId) throws ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticProfileSamples.class); - pstmt = con.prepareStatement("DELETE from " + - "genetic_profile_samples WHERE GENETIC_PROFILE_ID=?"); + pstmt = con.prepareStatement("DELETE from genetic_profile_samples WHERE genetic_profile_id=?"); pstmt.setLong(1, geneticProfileId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -117,11 +116,11 @@ public static ArrayList getOrderedSampleList(int geneticProfileId) thr try { con = JdbcUtil.getDbConnection(DaoGeneticProfileSamples.class); pstmt = con.prepareStatement - ("SELECT * FROM genetic_profile_samples WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT * FROM genetic_profile_samples WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); if (rs.next()) { - String orderedSampleList = rs.getString("ORDERED_SAMPLE_LIST"); + String orderedSampleList = rs.getString("ordered_sample_list"); // Split, based on DELIM token String[] parts = orderedSampleList.split(DELIM); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGistic.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGistic.java index 778db35e..cacc90ab 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGistic.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGistic.java @@ -61,6 +61,7 @@ public class DaoGistic { */ private static final Logger log = LoggerFactory.getLogger(DaoGistic.class); + private static final String GISTIC_SEQUENCE = "seq_gistic"; public static void addGistic(Gistic gistic) throws DaoException, validationException { if (gistic == null) { @@ -76,38 +77,34 @@ public static void addGistic(Gistic gistic) throws DaoException, validationExcep try { con = JdbcUtil.getDbConnection(DaoGistic.class); // insert into SQL gistic table + long roiId = ClickHouseAutoIncrement.nextId(GISTIC_SEQUENCE); pstmt = con.prepareStatement - ("INSERT INTO gistic (`CANCER_STUDY_ID`," + - "`CHROMOSOME`, " + - "`CYTOBAND`, " + - "`WIDE_PEAK_START`, " + - "`WIDE_PEAK_END`, " + - "`Q_VALUE`, " + - "`AMP`) " + - "VALUES (?,?,?,?,?,?,?)", - Statement.RETURN_GENERATED_KEYS); - - pstmt.setInt(1, gistic.getCancerStudyId()); - pstmt.setInt(2, gistic.getChromosome()) ; - pstmt.setString(3, gistic.getCytoband()); - pstmt.setInt(4, gistic.getPeakStart()); - pstmt.setInt(5, gistic.getPeakEnd()); - pstmt.setDouble(6, gistic.getqValue()); - pstmt.setBoolean(7, gistic.getAmp()); + ("INSERT INTO gistic (`gistic_roi_id`, `cancer_study_id`," + + "`chromosome`, " + + "`cytoband`, " + + "`wide_peak_start`, " + + "`wide_peak_end`, " + + "`q_value`, " + + "`amp`) " + + "VALUES (?,?,?,?,?,?,?,?)"); + + pstmt.setLong(1, roiId); + pstmt.setInt(2, gistic.getCancerStudyId()); + pstmt.setInt(3, gistic.getChromosome()) ; + pstmt.setString(4, gistic.getCytoband()); + pstmt.setInt(5, gistic.getPeakStart()); + pstmt.setInt(6, gistic.getPeakEnd()); + pstmt.setDouble(7, gistic.getqValue()); + pstmt.setBoolean(8, gistic.getAmp()); pstmt.executeUpdate(); - // insert into SQL gistic_to_gene table - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - int autoId = rs.getInt(1); - gistic.setInternalId(autoId); - } + gistic.setInternalId((int) roiId); addGisticGenes(gistic, con); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoGistic.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoGistic.class, con, pstmt, null); } } @@ -129,8 +126,8 @@ private static void addGisticGenes(Gistic gistic, Connection con) throws DaoExce // if this is the case, we are going to simply skip over this gene if (g.getEntrezGeneId() != -1) { pstmt = con.prepareStatement - ("INSERT INTO gistic_to_gene (`GISTIC_ROI_ID`," + - "`ENTREZ_GENE_ID`)" + + ("INSERT INTO gistic_to_gene (`gistic_roi_id`," + + "`entrez_gene_id`)" + "VALUES (?,?)"); pstmt.setInt(1, gistic.getInternalId()); @@ -176,28 +173,28 @@ private static Gistic extractGistic(Connection con, ResultSet rs) throws DaoExce try { - int id = rs.getInt("GISTIC_ROI_ID"); + int id = rs.getInt("gistic_roi_id"); - pstmt = con.prepareStatement("SELECT * FROM gistic_to_gene WHERE GISTIC_ROI_ID = ?"); + pstmt = con.prepareStatement("SELECT * FROM gistic_to_gene WHERE gistic_roi_id = ?"); pstmt.setInt(1, id); _rs = pstmt.executeQuery(); while ( _rs.next() ) { - long entrez = _rs.getLong("ENTREZ_GENE_ID"); + long entrez = _rs.getLong("entrez_gene_id"); CanonicalGene gene = DaoGeneOptimized.getInstance().getGene(entrez); genes.add(gene); } // create gistic return object - gistic = new Gistic(rs.getInt("CANCER_STUDY_ID"), - rs.getInt("CHROMOSOME") , - rs.getString("CYTOBAND") , - rs.getInt("WIDE_PEAK_START"), - rs.getInt("WIDE_PEAK_END"), - rs.getFloat("Q_VALUE") , + gistic = new Gistic(rs.getInt("cancer_study_id"), + rs.getInt("chromosome") , + rs.getString("cytoband") , + rs.getInt("wide_peak_start"), + rs.getInt("wide_peak_end"), + rs.getFloat("q_value") , genes, - rs.getBoolean("AMP")); + rs.getBoolean("amp")); } catch (SQLException e) { throw new DaoException(e); @@ -224,9 +221,9 @@ public static ArrayList getGisticByROI(int chromosome, int peakStart, in try { con = JdbcUtil.getDbConnection(DaoGistic.class); - pstmt = con.prepareStatement("SELECT * FROM gistic WHERE CHROMOSOME = ? " + - "AND WIDE_PEAK_START = ? " + - "AND WIDE_PEAK_END = ?"); + pstmt = con.prepareStatement("SELECT * FROM gistic WHERE chromosome = ? " + + "AND wide_peak_start = ? " + + "AND wide_peak_end = ?"); pstmt.setInt(1, chromosome); pstmt.setInt(2, peakStart); @@ -261,7 +258,7 @@ public static ArrayList getAllGisticByCancerStudyId(int cancerStudyId) t try { con = JdbcUtil.getDbConnection(DaoGistic.class); - pstmt = con.prepareStatement("SELECT * FROM gistic WHERE CANCER_STUDY_ID = ? "); + pstmt = con.prepareStatement("SELECT * FROM gistic WHERE cancer_study_id = ? "); pstmt.setInt(1, cancerStudyId); rs = pstmt.executeQuery(); @@ -294,7 +291,7 @@ public static int countGistic(int cancerStudy) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGistic.class); pstmt = con.prepareStatement - ("SELECT count(*) FROM gistic WHERE CANCER_STUDY_ID = ?"); + ("SELECT count(*) FROM gistic WHERE cancer_study_id = ?"); pstmt.setInt(1, cancerStudy); rs = pstmt.executeQuery(); @@ -352,11 +349,11 @@ public static void deleteGistic(int gisticInternalId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoGistic.class); - pstmt = con.prepareStatement("DELETE from gistic_to_gene WHERE GISTIC_ROI_ID=?"); + pstmt = con.prepareStatement("DELETE from gistic_to_gene WHERE gistic_roi_id=?"); pstmt.setInt(1, gisticInternalId); pstmt.executeUpdate(); - pstmt = con.prepareStatement("DELETE from gistic WHERE GISTIC_ROI_ID=?"); + pstmt = con.prepareStatement("DELETE from gistic WHERE gistic_roi_id=?"); pstmt.setInt(1, gisticInternalId); pstmt.executeUpdate(); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoInfo.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoInfo.java index dba1df6d..f76368e3 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoInfo.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoInfo.java @@ -56,7 +56,7 @@ public static synchronized void setVersion() { ("SELECT * FROM info"); rs = pstmt.executeQuery(); if (rs.next()) { - version = rs.getString("DB_SCHEMA_VERSION"); + version = rs.getString("db_schema_version"); } } catch (SQLException e) { e.printStackTrace(); @@ -98,7 +98,7 @@ public static void setGenesetVersion(String genesetVersion) throws DaoException connection = JdbcUtil.getDbConnection(DaoInfo.class); // Prepare SQL statement - preparedStatement = connection.prepareStatement("UPDATE info set GENESET_VERSION = ?"); + preparedStatement = connection.prepareStatement("ALTER TABLE info UPDATE geneset_version = ? WHERE 1"); preparedStatement.setString(1, genesetVersion); // Execute statement @@ -132,7 +132,7 @@ public static String getGenesetVersion() throws DaoException { // Extract version from result if (resultSet.next()) { - genesetVersion = resultSet.getString("GENESET_VERSION"); + genesetVersion = resultSet.getString("geneset_version"); } return genesetVersion; } catch (SQLException e) { @@ -153,7 +153,7 @@ public static void clearVersion() throws DaoException { try { con = JdbcUtil.getDbConnection(DaoInfo.class); - pstmt = con.prepareStatement("UPDATE info set GENESET_VERSION = NULL"); + pstmt = con.prepareStatement("ALTER TABLE info UPDATE geneset_version = NULL WHERE 1"); pstmt.executeUpdate(); } catch (SQLException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoMutSig.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoMutSig.java index 66df4f9c..b26183c5 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoMutSig.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoMutSig.java @@ -67,9 +67,9 @@ public static int addMutSig(MutSig mutSig) throws DaoException { CanonicalGene gene = mutSig.getCanonicalGene(); - if (MySQLbulkLoader.isBulkLoad()) { - // write to the temp file maintained by the MySQLbulkLoader - MySQLbulkLoader.getMySQLbulkLoader("mut_sig").insertRecord(Integer.toString(mutSig.getCancerType()), + if (ClickHouseBulkLoader.isBulkLoad()) { + // write to the temp file maintained by the ClickHouseBulkLoader + ClickHouseBulkLoader.getClickHouseBulkLoader("mut_sig").insertRecord(Integer.toString(mutSig.getCancerType()), Long.toString(gene.getEntrezGeneId()), Integer.toString(mutSig.getRank()), Integer.toString(mutSig.getNumBasesCovered()), @@ -89,13 +89,13 @@ public static int addMutSig(MutSig mutSig) throws DaoException { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("INSERT INTO mut_sig (`CANCER_STUDY_ID`," + - "`ENTREZ_GENE_ID`, " + - "`RANK`, " + + ("INSERT INTO mut_sig (`cancer_study_id`," + + "`entrez_gene_id`, " + + "`rank`, " + "`NumBasesCovered`, " + "`NumMutations`, " + - "`P_Value`, " + - "`Q_Value`) " + + "`p_value`, " + + "`q_value`) " + "VALUES (?,?,?,?,?,?,?)"); pstmt.setInt(1, mutSig.getCancerType()); @@ -136,7 +136,7 @@ public static MutSig getMutSig(String hugoGeneSymbol, int cancerStudy) throws Da try { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("SELECT * FROM mut_sig WHERE ENTREZ_GENE_ID = ? AND CANCER_STUDY_ID = ?"); + ("SELECT * FROM mut_sig WHERE entrez_gene_id = ? AND cancer_study_id = ?"); pstmt.setLong(1, entrezGeneID); pstmt.setInt(2, cancerStudy); rs = pstmt.executeQuery(); @@ -161,13 +161,13 @@ public static MutSig getMutSig(Long entrezGeneID, int cancerStudy) throws DaoExc try { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("SELECT * FROM mut_sig WHERE ENTREZ_GENE_ID = ? AND CANCER_STUDY_ID = ?"); + ("SELECT * FROM mut_sig WHERE entrez_gene_id = ? AND cancer_study_id = ?"); pstmt.setLong(1, entrezGeneID); pstmt.setInt(2, cancerStudy); rs = pstmt.executeQuery(); if (rs.next()) { //first go into gene database, and make a Canonical Gene Object with - CanonicalGene gene = daoGene.getGene(rs.getLong("ENTREZ_GENE_ID")); + CanonicalGene gene = daoGene.getGene(rs.getLong("entrez_gene_id")); return DaoMutSig.assignMutSig(gene, rs); } else { return null; @@ -189,12 +189,12 @@ public static ArrayList getAllMutSig(int cancerStudy) throws DaoExceptio try { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("SELECT * FROM mut_sig WHERE CANCER_STUDY_ID = ?"); + ("SELECT * FROM mut_sig WHERE cancer_study_id = ?"); pstmt.setInt(1, cancerStudy); rs = pstmt.executeQuery(); while (rs.next()) { - CanonicalGene gene = daoGene.getGene(rs.getLong("ENTREZ_GENE_ID")); + CanonicalGene gene = daoGene.getGene(rs.getLong("entrez_gene_id")); MutSig mutSig = DaoMutSig.assignMutSig(gene, rs); mutSigList.add(mutSig); } @@ -216,7 +216,7 @@ public static int countMutSig(int cancerStudy) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("SELECT count(*) FROM mut_sig WHERE CANCER_STUDY_ID = ?"); + ("SELECT count(*) FROM mut_sig WHERE cancer_study_id = ?"); pstmt.setInt(1, cancerStudy); rs = pstmt.executeQuery(); @@ -242,12 +242,12 @@ public static ArrayList getAllMutSig(int cancerStudy, double qValueThres try { con = JdbcUtil.getDbConnection(DaoMutSig.class); pstmt = con.prepareStatement - ("SELECT * FROM mut_sig WHERE CANCER_STUDY_ID = ? AND Q_Value < ?"); + ("SELECT * FROM mut_sig WHERE cancer_study_id = ? AND q_value < ?"); pstmt.setInt(1, cancerStudy); pstmt.setDouble(2,qValueThreshold); rs = pstmt.executeQuery(); while (rs.next()) { - CanonicalGene gene = daoGene.getGene(rs.getLong("ENTREZ_GENE_ID")); + CanonicalGene gene = daoGene.getGene(rs.getLong("entrez_gene_id")); MutSig mutSig = DaoMutSig.assignMutSig(gene, rs); mutSigList.add(mutSig); } @@ -277,13 +277,13 @@ public static void deleteAllRecords() throws DaoException { private static MutSig assignMutSig(CanonicalGene gene, ResultSet rs) throws SQLException, DaoException { - return new MutSig(rs.getInt("CANCER_STUDY_ID"), + return new MutSig(rs.getInt("cancer_study_id"), gene, - rs.getInt("RANK"), + rs.getInt("rank"), rs.getInt("NumBasesCovered"), - rs.getInt("numMutations"), - rs.getFloat("P_Value"), - rs.getFloat("Q_Value")); + rs.getInt("NumMutations"), + rs.getFloat("p_value"), + rs.getFloat("q_value")); } /** diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoMutation.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoMutation.java index 011783eb..1ae48f26 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoMutation.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoMutation.java @@ -32,9 +32,7 @@ package org.mskcc.cbio.portal.dao; -import java.sql.*; -import java.util.*; -import java.util.regex.*; +import org.apache.commons.collections4.MapIterator; import org.apache.commons.collections4.keyvalue.MultiKey; import org.apache.commons.collections4.map.MultiKeyMap; import org.apache.commons.collections4.MapIterator; @@ -48,18 +46,34 @@ import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.util.MutationKeywordUtils; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * Data access object for Mutation table */ public final class DaoMutation { public static final String NAN = "NaN"; private static final String MUTATION_COUNT_ATTR_ID = "MUTATION_COUNT"; - private static final String DELETE_ALTERATION_DRIVER_ANNOTATION = "DELETE from alteration_driver_annotation WHERE GENETIC_PROFILE_ID=? and SAMPLE_ID=?"; - private static final String DELETE_MUTATION = "DELETE from mutation WHERE GENETIC_PROFILE_ID=? and SAMPLE_ID=?"; + private static final String DELETE_ALTERATION_DRIVER_ANNOTATION = "DELETE from alteration_driver_annotation WHERE genetic_profile_id=? and sample_id=?"; + private static final String DELETE_MUTATION = "DELETE from mutation WHERE genetic_profile_id=? and sample_id=?"; public static int addMutation(ExtendedMutation mutation, boolean newMutationEvent) throws DaoException { - if (!MySQLbulkLoader.isBulkLoad()) { - throw new DaoException("You have to turn on MySQLbulkLoader in order to insert mutations"); + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to insert mutations"); } else { int result = 1; if (newMutationEvent) { @@ -74,7 +88,7 @@ public static int addMutation(ExtendedMutation mutation, boolean newMutationEven (mutation.getDriverTiersFilter() != null && !mutation.getDriverTiersFilter().isEmpty() && !mutation.getDriverTiersFilter().toLowerCase().equals("na"))) { - MySQLbulkLoader.getMySQLbulkLoader("alteration_driver_annotation").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("alteration_driver_annotation").insertRecord( Long.toString(mutation.getMutationEventId()), Integer.toString(mutation.getGeneticProfileId()), Integer.toString(mutation.getSampleId()), @@ -85,7 +99,7 @@ public static int addMutation(ExtendedMutation mutation, boolean newMutationEven ); } - MySQLbulkLoader.getMySQLbulkLoader("mutation").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("mutation").insertRecord( Long.toString(mutation.getMutationEventId()), Integer.toString(mutation.getGeneticProfileId()), Integer.toString(mutation.getSampleId()), @@ -122,9 +136,9 @@ public static int addMutation(ExtendedMutation mutation, boolean newMutationEven public static int addMutationEvent(ExtendedMutation.MutationEvent event) throws DaoException { // use this code if bulk loading - // write to the temp file maintained by the MySQLbulkLoader + // write to the temp file maintained by the ClickHouseBulkLoader String keyword = MutationKeywordUtils.guessOncotatorMutationKeyword(event.getProteinChange(), event.getMutationType()); - MySQLbulkLoader.getMySQLbulkLoader("mutation_event").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("mutation_event").insertRecord( Long.toString(event.getMutationEventId()), Long.toString(event.getGene().getEntrezGeneId()), event.getChr(), @@ -151,7 +165,8 @@ public static int addMutationEvent(ExtendedMutation.MutationEvent event) throws public static void createMutationCountClinicalData(GeneticProfile geneticProfile) throws DaoException { Connection con = null; - PreparedStatement pstmt = null; + PreparedStatement deleteStmt = null; + PreparedStatement insertStmt = null; ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); @@ -163,47 +178,42 @@ public static void createMutationCountClinicalData(GeneticProfile geneticProfile DaoClinicalAttributeMeta.addDatum(attr); } - /* - * Add MUTATION_COUNT for each sample by checking number of - * mutations for the given genetic profile. - * - * We do not add the MUTATION_COUNT clinical data for the sample if - * it's not profiled. If it *is* profiled but there are 0 - * mutations, add a MUTATION_COUNT with 0 value record. Do not - * include germline when counting mutations. - * - * Use REPLACE (conditional INSERT/UPDATE) which inserts - * new counts if they don't exist and overwrites them if they do. - * This is necessary for when the mutation data is split over - * multiple files with same profile id. Note that since - * clinical_sample has a key contraint on INTERNAL_ID and ATTR_ID, - * there can only be one MUTATION_COUNT record for each sample, so - * we assume each sample is in only one MUTATION_EXTENDED profile. - */ - pstmt = con.prepareStatement( - "REPLACE `clinical_sample` " + - "SELECT sample_profile.`SAMPLE_ID`, 'MUTATION_COUNT', COUNT(DISTINCT mutation_event.`CHR`, mutation_event.`START_POSITION`, " + - "mutation_event.`END_POSITION`, mutation_event.`REFERENCE_ALLELE`, mutation_event.`TUMOR_SEQ_ALLELE`) AS MUTATION_COUNT " + + String mutationCountSelect = + "SELECT sample_profile.`sample_id`, 'MUTATION_COUNT', count(DISTINCT mutation_event.`chr`, mutation_event.`start_position`, " + + "mutation_event.`end_position`, mutation_event.`reference_allele`, mutation_event.`tumor_seq_allele`) AS MUTATION_COUNT " + "FROM `sample_profile` " + - "LEFT JOIN mutation ON mutation.`SAMPLE_ID` = sample_profile.`SAMPLE_ID` " + - "AND ( mutation.`MUTATION_STATUS` <> 'GERMLINE' OR mutation.`MUTATION_STATUS` IS NULL ) " + - "LEFT JOIN mutation_event ON mutation.`MUTATION_EVENT_ID` = mutation_event.`MUTATION_EVENT_ID` " + - "INNER JOIN genetic_profile ON genetic_profile.`GENETIC_PROFILE_ID` = sample_profile.`GENETIC_PROFILE_ID` " + - "WHERE genetic_profile.`GENETIC_ALTERATION_TYPE` = 'MUTATION_EXTENDED' " + - "AND genetic_profile.`GENETIC_PROFILE_ID`=? " + - "GROUP BY sample_profile.`GENETIC_PROFILE_ID` , sample_profile.`SAMPLE_ID`;"); - pstmt.setInt(1, geneticProfile.getGeneticProfileId()); - pstmt.executeUpdate(); + "LEFT JOIN mutation ON mutation.`sample_id` = sample_profile.`sample_id` " + + "AND ( UPPER(mutation.`mutation_status`) <> 'GERMLINE' OR mutation.`mutation_status` IS NULL ) " + + "LEFT JOIN mutation_event ON mutation.`mutation_event_id` = mutation_event.`mutation_event_id` " + + "INNER JOIN genetic_profile ON genetic_profile.`genetic_profile_id` = sample_profile.`genetic_profile_id` " + + "WHERE genetic_profile.`genetic_alteration_type` = 'MUTATION_EXTENDED' " + + "AND genetic_profile.`genetic_profile_id`=? " + + "GROUP BY sample_profile.`genetic_profile_id` , sample_profile.`sample_id`"; + + deleteStmt = con.prepareStatement( + "ALTER TABLE clinical_sample " + + "DELETE WHERE attr_id = 'MUTATION_COUNT' " + + "AND internal_id IN (SELECT sample_id FROM sample_profile WHERE genetic_profile_id = ?) " + ); + deleteStmt.setInt(1, geneticProfile.getGeneticProfileId()); + deleteStmt.executeUpdate(); + + insertStmt = con.prepareStatement( + "INSERT INTO clinical_sample " + mutationCountSelect + ); + insertStmt.setInt(1, geneticProfile.getGeneticProfileId()); + insertStmt.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoMutation.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoMutation.class, null, deleteStmt, null); + JdbcUtil.closeAll(DaoMutation.class, con, insertStmt, rs); } } public static void calculateMutationCountByKeyword(int geneticProfileId) throws DaoException { - if (!MySQLbulkLoader.isBulkLoad()) { - throw new DaoException("You have to turn on MySQLbulkLoader in order to update mutation counts by keyword"); + if (!ClickHouseBulkLoader.isBulkLoad()) { + throw new DaoException("You have to turn on ClickHouseBulkLoader in order to update mutation counts by keyword"); } else { MultiKeyMap mutationEventKeywordCountMap = getMutationEventKeywordCountByGeneticProfileId(geneticProfileId); // mutation event keyword -> entrez id -> keyword count Map geneCountMap = getGeneCountByGeneticProfileId(geneticProfileId); // entrez id -> gene count @@ -215,7 +225,7 @@ public static void calculateMutationCountByKeyword(int geneticProfileId) throws Long entrezGeneId = Long.valueOf(mk.getKey(1).toString()); String keywordCount = it.getValue().toString(); Integer geneCount = geneCountMap.get(entrezGeneId); - MySQLbulkLoader.getMySQLbulkLoader("mutation_count_by_keyword").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("mutation_count_by_keyword").insertRecord( Integer.toString(geneticProfileId), mutationEventKeyword, Long.toString(entrezGeneId), @@ -234,10 +244,10 @@ public static MultiKeyMap getMutationEventKeywordCountByGeneticProfileId(int gen try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( - "SELECT mutation_event.`KEYWORD`, mutation_event.`ENTREZ_GENE_ID`, IF(mutation_event.`KEYWORD` IS NULL, 0, COUNT(DISTINCT(mutation.SAMPLE_ID))) AS KEYWORD_COUNT " + - "FROM mutation_event JOIN mutation on mutation.`MUTATION_EVENT_ID` = mutation_event.`MUTATION_EVENT_ID` " + - "WHERE mutation.`GENETIC_PROFILE_ID` = ? " + - "GROUP BY mutation_event.`KEYWORD`, mutation_event.`ENTREZ_GENE_ID`;" + "SELECT mutation_event.`keyword`, mutation_event.`entrez_gene_id`, if(mutation_event.`keyword` IS NULL, 0, count(DISTINCT(mutation.sample_id))) AS keyword_count " + + "FROM mutation_event JOIN mutation on mutation.`mutation_event_id` = mutation_event.`mutation_event_id` " + + "WHERE mutation.`genetic_profile_id` = ? " + + "GROUP BY mutation_event.`keyword`, mutation_event.`entrez_gene_id`" ); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); @@ -260,13 +270,13 @@ public static Map getGeneCountByGeneticProfileId(int geneticProfi try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( - "SELECT ENTREZ_GENE_ID AS `ENTREZ_GENE_ID`, COUNT(DISTINCT(SAMPLE_ID)) AS `GENE_COUNT`" + - " FROM mutation WHERE GENETIC_PROFILE_ID = ? " + - "GROUP BY ENTREZ_GENE_ID;"); + "SELECT entrez_gene_id AS `entrez_gene_id`, count(DISTINCT(sample_id)) AS `gene_count`" + + " FROM mutation WHERE genetic_profile_id = ? " + + "GROUP BY entrez_gene_id"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { - geneCountByGeneticProfileId.put(rs.getLong("ENTREZ_GENE_ID"), rs.getInt("GENE_COUNT")); + geneCountByGeneticProfileId.put(rs.getLong("entrez_gene_id"), rs.getInt("gene_count")); } return geneCountByGeneticProfileId; } catch (SQLException e) { @@ -277,7 +287,7 @@ public static Map getGeneCountByGeneticProfileId(int geneticProfi } /** - * Used by GeneticAlterationUtil (which is used by GetProfileData). This use comes out of an + * Used by GeneticAlterationUtil (which is used by GetProfileData via webservice.do). This use comes out of an * effort to discontinue the use of the business module. */ public static ArrayList getMutations (int geneticProfileId, Collection targetSampleList, @@ -290,9 +300,9 @@ public static ArrayList getMutations (int geneticProfileId, Co con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE SAMPLE_ID IN ('" + org.apache.commons.lang3.StringUtils.join(targetSampleList, "','") + - "') AND GENETIC_PROFILE_ID = ? AND mutation.ENTREZ_GENE_ID = ?"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE sample_id IN ('" + org.apache.commons.lang3.StringUtils.join(targetSampleList, "','") + + "') AND genetic_profile_id = ? AND mutation.entrez_gene_id = ?"); pstmt.setInt(1, geneticProfileId); pstmt.setLong(2, entrezGeneId); rs = pstmt.executeQuery(); @@ -321,16 +331,16 @@ public static HashMap getSimplifiedMutations (int geneticProfileId, Collection getMutations (int geneticProfileId, in con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE SAMPLE_ID = ? AND GENETIC_PROFILE_ID = ? AND mutation.ENTREZ_GENE_ID = ?"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE sample_id = ? AND genetic_profile_id = ? AND mutation.entrez_gene_id = ?"); pstmt.setInt(1, sampleId); pstmt.setInt(2, geneticProfileId); pstmt.setLong(3, entrezGeneId); @@ -391,11 +401,11 @@ public static Set getGenesInProfile(int geneticProfileId) throws try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( - "SELECT DISTINCT ENTREZ_GENE_ID FROM mutation WHERE GENETIC_PROFILE_ID = ?"); + "SELECT DISTINCT entrez_gene_id FROM mutation WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { - geneSet.add(daoGene.getGene(rs.getLong("ENTREZ_GENE_ID"))); + geneSet.add(daoGene.getGene(rs.getLong("entrez_gene_id"))); } } catch (SQLException e) { throw new DaoException(e); @@ -418,8 +428,8 @@ public static ArrayList getMutations (long entrezGeneId) throw con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE mutation.ENTREZ_GENE_ID = ?"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE mutation.entrez_gene_id = ?"); pstmt.setLong(1, entrezGeneId); rs = pstmt.executeQuery(); while (rs.next()) { @@ -447,8 +457,8 @@ public static ArrayList getMutations (long entrezGeneId, Strin con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation_event " + - "INNER JOIN mutation ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE mutation.ENTREZ_GENE_ID = ? AND PROTEIN_CHANGE = ?"); + "INNER JOIN mutation ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE mutation.entrez_gene_id = ? AND protein_change = ?"); pstmt.setLong(1, entrezGeneId); pstmt.setString(2, aminoAcidChange); rs = pstmt.executeQuery(); @@ -477,8 +487,8 @@ public static ArrayList getMutations (int geneticProfileId, Li con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE GENETIC_PROFILE_ID = ? AND SAMPLE_ID in ('"+ StringUtils.join(sampleIds, "','")+"')"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE genetic_profile_id = ? AND sample_id in ('"+ StringUtils.join(sampleIds, "','")+"')"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { @@ -505,7 +515,7 @@ public static boolean hasAlleleFrequencyData (int geneticProfileId, int sampleId con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT EXISTS (SELECT 1 FROM mutation " + - "WHERE GENETIC_PROFILE_ID = ? AND SAMPLE_ID = ? AND TUMOR_ALT_COUNT>=0 AND TUMOR_REF_COUNT>=0)"); + "WHERE genetic_profile_id = ? AND sample_id = ? AND tumor_alt_count>=0 AND tumor_ref_count>=0)"); pstmt.setInt(1, geneticProfileId); pstmt.setInt(2, sampleId); rs = pstmt.executeQuery(); @@ -532,8 +542,8 @@ public static ArrayList getMutations (long entrezGeneId, Strin con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation, mutation_event " + - "WHERE mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "AND mutation.ENTREZ_GENE_ID = ? AND PROTEIN_CHANGE = ? AND SAMPLE_ID <> ?"); + "WHERE mutation.mutation_event_id=mutation_event.mutation_event_id " + + "AND mutation.entrez_gene_id = ? AND protein_change = ? AND sample_id <> ?"); pstmt.setLong(1, entrezGeneId); pstmt.setString(2, aminoAcidChange); pstmt.setInt(3, excludeSampleId); @@ -566,8 +576,8 @@ public static ArrayList getMutations (int geneticProfileId, con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE GENETIC_PROFILE_ID = ? AND mutation.ENTREZ_GENE_ID = ?"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE genetic_profile_id = ? AND mutation.entrez_gene_id = ?"); pstmt.setInt(1, geneticProfileId); pstmt.setLong(2, entrezGeneId); rs = pstmt.executeQuery(); @@ -592,7 +602,7 @@ public static ArrayList getAllMutations () throws DaoException con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id"); rs = pstmt.executeQuery(); while (rs.next()) { ExtendedMutation mutation = extractMutation(rs); @@ -624,8 +634,8 @@ public static ArrayList getAllMutations (int geneticProfileId) con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT * FROM mutation " + - "INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE mutation.GENETIC_PROFILE_ID = ?"); + "INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE mutation.genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); while (rs.next()) { @@ -640,10 +650,6 @@ public static ArrayList getAllMutations (int geneticProfileId) return mutationList; } - /** - * @deprecated We believe that this method is no longer called by any part of the codebase, and it will soon be deleted. - */ - @Deprecated public static Set getAllMutationEvents() throws DaoException { Connection con = null; PreparedStatement pstmt = null; @@ -665,51 +671,13 @@ public static Set getAllMutationEvents() throws return events; } - /* - * Returns an existing MutationEvent record from the database or null. - */ - public static ExtendedMutation.MutationEvent getMutationEvent(ExtendedMutation.MutationEvent event) throws DaoException { - Connection con = null; - PreparedStatement pstmt = null; - ResultSet rs = null; - try { - con = JdbcUtil.getDbConnection(DaoMutation.class); - pstmt = con.prepareStatement("SELECT * from mutation_event WHERE" + - " `ENTREZ_GENE_ID`=?" + - " AND `CHR`=?" + - " AND `START_POSITION`=?" + - " AND `END_POSITION`=?" + - " AND `TUMOR_SEQ_ALLELE`=?" + - " AND `PROTEIN_CHANGE`=?" + - " AND `MUTATION_TYPE`=?"); - pstmt.setLong(1, event.getGene().getEntrezGeneId()); - pstmt.setString(2, event.getChr()); - pstmt.setLong(3, event.getStartPosition()); - pstmt.setLong(4, event.getEndPosition()); - pstmt.setString(5, event.getTumorSeqAllele()); - pstmt.setString(6, event.getProteinChange()); - pstmt.setString(7, event.getMutationType()); - rs = pstmt.executeQuery(); - if (rs.next()) { - return extractMutationEvent(rs); - } - else { - return null; - } - } catch (SQLException e) { - throw new DaoException(e); - } finally { - JdbcUtil.closeAll(DaoMutation.class, con, pstmt, rs); - } - } - public static long getLargestMutationEventId() throws DaoException { Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - pstmt = con.prepareStatement("SELECT MAX(`MUTATION_EVENT_ID`) FROM `mutation_event`"); + pstmt = con.prepareStatement("SELECT max(`mutation_event_id`) FROM `mutation_event`"); rs = pstmt.executeQuery(); return rs.next() ? rs.getLong(1) : 0; } catch (SQLException e) { @@ -722,39 +690,39 @@ public static long getLargestMutationEventId() throws DaoException { private static ExtendedMutation extractMutation(ResultSet rs) throws SQLException, DaoException { try { ExtendedMutation mutation = new ExtendedMutation(extractMutationEvent(rs)); - mutation.setGeneticProfileId(rs.getInt("GENETIC_PROFILE_ID")); - mutation.setSampleId(rs.getInt("SAMPLE_ID")); - mutation.setSequencingCenter(rs.getString("CENTER")); - mutation.setSequencer(rs.getString("SEQUENCER")); - mutation.setMutationStatus(rs.getString("MUTATION_STATUS")); - mutation.setValidationStatus(rs.getString("VALIDATION_STATUS")); - mutation.setTumorSeqAllele1(rs.getString("TUMOR_SEQ_ALLELE1")); - mutation.setTumorSeqAllele2(rs.getString("TUMOR_SEQ_ALLELE2")); - mutation.setMatchedNormSampleBarcode(rs.getString("MATCHED_NORM_SAMPLE_BARCODE")); - mutation.setMatchNormSeqAllele1(rs.getString("MATCH_NORM_SEQ_ALLELE1")); - mutation.setMatchNormSeqAllele2(rs.getString("MATCH_NORM_SEQ_ALLELE2")); - mutation.setTumorValidationAllele1(rs.getString("TUMOR_VALIDATION_ALLELE1")); - mutation.setTumorValidationAllele2(rs.getString("TUMOR_VALIDATION_ALLELE2")); - mutation.setMatchNormValidationAllele1(rs.getString("MATCH_NORM_VALIDATION_ALLELE1")); - mutation.setMatchNormValidationAllele2(rs.getString("MATCH_NORM_VALIDATION_ALLELE2")); - mutation.setVerificationStatus(rs.getString("VERIFICATION_STATUS")); - mutation.setSequencingPhase(rs.getString("SEQUENCING_PHASE")); - mutation.setSequenceSource(rs.getString("SEQUENCE_SOURCE")); - mutation.setValidationMethod(rs.getString("VALIDATION_METHOD")); - mutation.setScore(rs.getString("SCORE")); - mutation.setBamFile(rs.getString("BAM_FILE")); + mutation.setGeneticProfileId(rs.getInt("genetic_profile_id")); + mutation.setSampleId(rs.getInt("sample_id")); + mutation.setSequencingCenter(rs.getString("center")); + mutation.setSequencer(rs.getString("sequencer")); + mutation.setMutationStatus(rs.getString("mutation_status")); + mutation.setValidationStatus(rs.getString("validation_status")); + mutation.setTumorSeqAllele1(rs.getString("tumor_seq_allele1")); + mutation.setTumorSeqAllele2(rs.getString("tumor_seq_allele2")); + mutation.setMatchedNormSampleBarcode(rs.getString("matched_norm_sample_barcode")); + mutation.setMatchNormSeqAllele1(rs.getString("match_norm_seq_allele1")); + mutation.setMatchNormSeqAllele2(rs.getString("match_norm_seq_allele2")); + mutation.setTumorValidationAllele1(rs.getString("tumor_validation_allele1")); + mutation.setTumorValidationAllele2(rs.getString("tumor_validation_allele2")); + mutation.setMatchNormValidationAllele1(rs.getString("match_norm_validation_allele1")); + mutation.setMatchNormValidationAllele2(rs.getString("match_norm_validation_allele2")); + mutation.setVerificationStatus(rs.getString("verification_status")); + mutation.setSequencingPhase(rs.getString("sequencing_phase")); + mutation.setSequenceSource(rs.getString("sequence_source")); + mutation.setValidationMethod(rs.getString("validation_method")); + mutation.setScore(rs.getString("score")); + mutation.setBamFile(rs.getString("bam_file")); // the following checks/set of null is to maintain // behavior with MutationRepositoryLegacy (mybatis) behavior // whose use is being retired in this PR - mutation.setTumorAltCount(rs.getInt("TUMOR_ALT_COUNT")); + mutation.setTumorAltCount(rs.getInt("tumor_alt_count")); if (rs.wasNull()) mutation.setTumorAltCount(null); - mutation.setTumorRefCount(rs.getInt("TUMOR_REF_COUNT")); + mutation.setTumorRefCount(rs.getInt("tumor_ref_count")); if (rs.wasNull()) mutation.setTumorRefCount(null); - mutation.setNormalAltCount(rs.getInt("NORMAL_ALT_COUNT")); + mutation.setNormalAltCount(rs.getInt("normal_alt_count")); if (rs.wasNull()) mutation.setNormalAltCount(null); - mutation.setNormalRefCount(rs.getInt("NORMAL_REF_COUNT")); + mutation.setNormalRefCount(rs.getInt("normal_ref_count")); if (rs.wasNull()) mutation.setNormalRefCount(null); - mutation.setAnnotationJson(rs.getString("ANNOTATION_JSON")); + mutation.setAnnotationJson(rs.getString("annotation_json")); return mutation; } catch(NullPointerException e) { @@ -764,30 +732,30 @@ private static ExtendedMutation extractMutation(ResultSet rs) throws SQLExceptio private static ExtendedMutation.MutationEvent extractMutationEvent(ResultSet rs) throws SQLException, DaoException { ExtendedMutation.MutationEvent event = new ExtendedMutation.MutationEvent(); - event.setMutationEventId(rs.getLong("MUTATION_EVENT_ID")); - long entrezId = rs.getLong("mutation_event.ENTREZ_GENE_ID"); + event.setMutationEventId(rs.getLong("mutation_event_id")); + long entrezId = rs.getLong("entrez_gene_id"); DaoGeneOptimized aDaoGene = DaoGeneOptimized.getInstance(); CanonicalGene gene = aDaoGene.getGene(entrezId); event.setGene(gene); - event.setChr(rs.getString("CHR")); - event.setStartPosition(rs.getLong("START_POSITION")); - event.setEndPosition(rs.getLong("END_POSITION")); - event.setProteinChange(rs.getString("PROTEIN_CHANGE")); - event.setMutationType(rs.getString("MUTATION_TYPE")); - event.setNcbiBuild(rs.getString("NCBI_BUILD")); - event.setStrand(rs.getString("STRAND")); - event.setVariantType(rs.getString("VARIANT_TYPE")); - event.setDbSnpRs(rs.getString("DB_SNP_RS")); - event.setDbSnpValStatus(rs.getString("DB_SNP_VAL_STATUS")); - event.setReferenceAllele(rs.getString("REFERENCE_ALLELE")); - event.setRefseqMrnaId(rs.getString("REFSEQ_MRNA_ID")); - event.setCodonChange(rs.getString("CODON_CHANGE")); - event.setUniprotAccession(rs.getString("UNIPROT_ACCESSION")); - event.setProteinPosStart(rs.getInt("PROTEIN_POS_START")); - event.setProteinPosEnd(rs.getInt("PROTEIN_POS_END")); - event.setCanonicalTranscript(rs.getBoolean("CANONICAL_TRANSCRIPT")); - event.setTumorSeqAllele(rs.getString("TUMOR_SEQ_ALLELE")); - event.setKeyword(rs.getString("KEYWORD")); + event.setChr(rs.getString("chr")); + event.setStartPosition(rs.getLong("start_position")); + event.setEndPosition(rs.getLong("end_position")); + event.setProteinChange(rs.getString("protein_change")); + event.setMutationType(rs.getString("mutation_type")); + event.setNcbiBuild(rs.getString("ncbi_build")); + event.setStrand(rs.getString("strand")); + event.setVariantType(rs.getString("variant_type")); + event.setDbSnpRs(rs.getString("db_snp_rs")); + event.setDbSnpValStatus(rs.getString("db_snp_val_status")); + event.setReferenceAllele(rs.getString("reference_allele")); + event.setRefseqMrnaId(rs.getString("refseq_mrna_id")); + event.setCodonChange(rs.getString("codon_change")); + event.setUniprotAccession(rs.getString("uniprot_accession")); + event.setProteinPosStart(rs.getInt("protein_pos_start")); + event.setProteinPosEnd(rs.getInt("protein_pos_end")); + event.setCanonicalTranscript(rs.getBoolean("canonical_transcript")); + event.setTumorSeqAllele(rs.getString("tumor_seq_allele")); + event.setKeyword(rs.getString("keyword")); return event; } @@ -801,7 +769,7 @@ public static int getCount() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - pstmt = con.prepareStatement("SELECT COUNT(*) FROM mutation"); + pstmt = con.prepareStatement("SELECT count(*) FROM mutation"); rs = pstmt.executeQuery(); if (rs.next()) { return rs.getInt(1); @@ -836,13 +804,13 @@ public static Map> getSMGs(int profileId, Collection0?(" HAVING COUNT(*)>="+thresholdRecurrence):"") + " ORDER BY count_per_nt DESC" + (thresholdNumGenes>0?(" LIMIT 0,"+thresholdNumGenes):""); @@ -878,8 +846,8 @@ public static int countMutationEvents(int profileId) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT count(DISTINCT `SAMPLE_ID`, `MUTATION_EVENT_ID`) FROM mutation" + - " WHERE `GENETIC_PROFILE_ID`=" + profileId; + String sql = "SELECT count(DISTINCT `sample_id`, `mutation_event_id`) FROM mutation" + + " WHERE `genetic_profile_id`=" + profileId; pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); if (rs.next()) { @@ -923,15 +891,15 @@ public static Map> getSamplesWithMutations(String concatEvent ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT `SAMPLE_ID`, `MUTATION_EVENT_ID` FROM mutation" + - " WHERE `MUTATION_EVENT_ID` IN (" + + String sql = "SELECT `sample_id`, `mutation_event_id` FROM mutation" + + " WHERE `mutation_event_id` IN (" + concatEventIds + ")"; pstmt = con.prepareStatement(sql); Map> map = new HashMap> (); rs = pstmt.executeQuery(); while (rs.next()) { - int sampleId = rs.getInt("SAMPLE_ID"); - long eventId = rs.getLong("MUTATION_EVENT_ID"); + int sampleId = rs.getInt("sample_id"); + long eventId = rs.getLong("mutation_event_id"); Set events = map.get(sampleId); if (events == null) { events = new HashSet(); @@ -975,17 +943,17 @@ public static Map> getSimilarSamplesWithMutationsByKeywords( ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT `SAMPLE_ID`, `GENETIC_PROFILE_ID`, me1.`MUTATION_EVENT_ID`" + + String sql = "SELECT `sample_id`, `genetic_profile_id`, me1.`mutation_event_id`" + " FROM mutation cme, mutation_event me1, mutation_event me2" + - " WHERE me1.`MUTATION_EVENT_ID` IN ("+ concatEventIds + ")" + - " AND me1.`KEYWORD`=me2.`KEYWORD`" + - " AND cme.`MUTATION_EVENT_ID`=me2.`MUTATION_EVENT_ID`"; + " WHERE me1.`mutation_event_id` IN ("+ concatEventIds + ")" + + " AND me1.`keyword`=me2.`keyword`" + + " AND cme.`mutation_event_id`=me2.`mutation_event_id`"; pstmt = con.prepareStatement(sql); Map> map = new HashMap> (); rs = pstmt.executeQuery(); while (rs.next()) { - Sample sample = DaoSample.getSampleById(rs.getInt("SAMPLE_ID")); - long eventId = rs.getLong("MUTATION_EVENT_ID"); + Sample sample = DaoSample.getSampleById(rs.getInt("sample_id")); + long eventId = rs.getLong("mutation_event_id"); Set events = map.get(sample); if (events == null) { events = new HashSet(); @@ -1021,15 +989,15 @@ public static Map> getSimilarSamplesWithMutatedGenes( ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT `SAMPLE_ID`, `GENETIC_PROFILE_ID`, `ENTREZ_GENE_ID`" + + String sql = "SELECT `sample_id`, `genetic_profile_id`, `entrez_gene_id`" + " FROM mutation" + - " WHERE `ENTREZ_GENE_ID` IN ("+ StringUtils.join(entrezGeneIds,",") + ")"; + " WHERE `entrez_gene_id` IN ("+ StringUtils.join(entrezGeneIds,",") + ")"; pstmt = con.prepareStatement(sql); Map> map = new HashMap> (); rs = pstmt.executeQuery(); while (rs.next()) { - Sample sample = DaoSample.getSampleById(rs.getInt("SAMPLE_ID")); - long entrez = rs.getLong("ENTREZ_GENE_ID"); + Sample sample = DaoSample.getSampleById(rs.getInt("sample_id")); + long entrez = rs.getLong("entrez_gene_id"); Set genes = map.get(sample); if (genes == null) { genes = new HashSet(); @@ -1074,11 +1042,11 @@ public static Map countSamplesWithMutationEvents(String concatEve ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT `MUTATION_EVENT_ID`, count(DISTINCT `SAMPLE_ID`) FROM mutation" + - " WHERE `GENETIC_PROFILE_ID`=" + profileId + - " AND `MUTATION_EVENT_ID` IN (" + + String sql = "SELECT `mutation_event_id`, count(DISTINCT `sample_id`) FROM mutation" + + " WHERE `genetic_profile_id`=" + profileId + + " AND `mutation_event_id` IN (" + concatEventIds + - ") GROUP BY `MUTATION_EVENT_ID`"; + ") GROUP BY `mutation_event_id`"; pstmt = con.prepareStatement(sql); Map map = new HashMap(); rs = pstmt.executeQuery(); @@ -1119,12 +1087,12 @@ public static Map countSamplesWithMutatedGenes(String concatEntre ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID)" + + String sql = "SELECT entrez_gene_id, count(DISTINCT sample_id)" + " FROM mutation" + - " WHERE GENETIC_PROFILE_ID=" + profileId + - " AND ENTREZ_GENE_ID IN (" + + " WHERE genetic_profile_id=" + profileId + + " AND entrez_gene_id IN (" + concatEntrezGeneIds + - ") GROUP BY `ENTREZ_GENE_ID`"; + ") GROUP BY `entrez_gene_id`"; pstmt = con.prepareStatement(sql); Map map = new HashMap(); rs = pstmt.executeQuery(); @@ -1152,13 +1120,13 @@ public static Map countSamplesWithKeywords(Collection k ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT KEYWORD, count(DISTINCT SAMPLE_ID)" + + String sql = "SELECT keyword, count(DISTINCT sample_id)" + " FROM mutation, mutation_event" + - " WHERE GENETIC_PROFILE_ID=" + profileId + - " AND mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID" + - " AND KEYWORD IN ('" + + " WHERE genetic_profile_id=" + profileId + + " AND mutation.mutation_event_id=mutation_event.mutation_event_id" + + " AND keyword IN ('" + StringUtils.join(keywords,"','") + - "') GROUP BY `KEYWORD`"; + "') GROUP BY `keyword`"; pstmt = con.prepareStatement(sql); Map map = new HashMap(); rs = pstmt.executeQuery(); @@ -1189,11 +1157,11 @@ public static Collection> countSamplesWithKeywords(Collectio ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT KEYWORD, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) FROM mutation, mutation_event " + - "WHERE GENETIC_PROFILE_ID IN (" + StringUtils.join(internalProfileIds, ",") + ") " + - "AND mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "AND KEYWORD IN ('" + StringUtils.join(keywords, "','") + "') " + - "GROUP BY KEYWORD, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID"; + String sql = "SELECT keyword, genetic_profile_id, mutation.entrez_gene_id, count(DISTINCT sample_id) FROM mutation, mutation_event " + + "WHERE genetic_profile_id IN (" + StringUtils.join(internalProfileIds, ",") + ") " + + "AND mutation.mutation_event_id=mutation_event.mutation_event_id " + + "AND keyword IN ('" + StringUtils.join(keywords, "','") + "') " + + "GROUP BY keyword, genetic_profile_id, mutation.entrez_gene_id"; pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); Collection> data = new ArrayList>(); @@ -1255,11 +1223,11 @@ public static Collection> countSamplesWithGenes(Collection> data = new ArrayList>(); @@ -1299,11 +1267,11 @@ public static Collection> countSamplesWithProteinChanges( ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT PROTEIN_CHANGE, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) FROM mutation, mutation_event " + - "WHERE GENETIC_PROFILE_ID IN (" + StringUtils.join(internalProfileIds, ",") + ") " + - "AND mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "AND PROTEIN_CHANGE IN ('" + StringUtils.join(proteinChanges, "','") + "') " + - "GROUP BY PROTEIN_CHANGE, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID"; + String sql = "SELECT protein_change, genetic_profile_id, mutation.entrez_gene_id, count(DISTINCT sample_id) FROM mutation, mutation_event " + + "WHERE genetic_profile_id IN (" + StringUtils.join(internalProfileIds, ",") + ") " + + "AND mutation.mutation_event_id=mutation_event.mutation_event_id " + + "AND protein_change IN ('" + StringUtils.join(proteinChanges, "','") + "') " + + "GROUP BY protein_change, genetic_profile_id, mutation.entrez_gene_id"; pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); Collection> data = new ArrayList>(); @@ -1358,12 +1326,12 @@ public static Collection> countSamplesWithProteinPosStarts( if (geneIdSet.size() == 0 || internalProfileIds.size() == 0) return new ArrayList>(); //empty IN() clause would be a SQL error below try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID, count(DISTINCT SAMPLE_ID) " + - "FROM mutation INNER JOIN mutation_event ON mutation.MUTATION_EVENT_ID=mutation_event.MUTATION_EVENT_ID " + - "WHERE mutation.ENTREZ_GENE_ID IN (" + StringUtils.join(geneIdSet, ",") + ") " + - "AND GENETIC_PROFILE_ID IN (" + StringUtils.join(internalProfileIds, ",") + ") " + - "AND (mutation.ENTREZ_GENE_ID, PROTEIN_POS_START) IN (" + StringUtils.join(proteinPosStarts, ",") + ") " + - "GROUP BY PROTEIN_POS_START, GENETIC_PROFILE_ID, mutation.ENTREZ_GENE_ID"; + String sql = "SELECT protein_pos_start, genetic_profile_id, mutation.entrez_gene_id, count(DISTINCT sample_id) " + + "FROM mutation INNER JOIN mutation_event ON mutation.mutation_event_id=mutation_event.mutation_event_id " + + "WHERE mutation.entrez_gene_id IN (" + StringUtils.join(geneIdSet, ",") + ") " + + "AND genetic_profile_id IN (" + StringUtils.join(internalProfileIds, ",") + ") " + + "AND (mutation.entrez_gene_id, protein_pos_start) IN (" + StringUtils.join(proteinPosStarts, ",") + ") " + + "GROUP BY protein_pos_start, genetic_profile_id, mutation.entrez_gene_id"; pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); Collection> data = new ArrayList>(); @@ -1426,8 +1394,8 @@ public static Set getGenesOfMutations(String concatEventIds, int profileId ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT DISTINCT ENTREZ_GENE_ID FROM mutation_event " + - "WHERE MUTATION_EVENT_ID in (" + concatEventIds + ")"; + String sql = "SELECT DISTINCT entrez_gene_id FROM mutation_event " + + "WHERE mutation_event_id in (" + concatEventIds + ")"; pstmt = con.prepareStatement(sql); Set set = new HashSet(); rs = pstmt.executeQuery(); @@ -1461,8 +1429,8 @@ public static Set getKeywordsOfMutations(String concatEventIds, int prof ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - String sql = "SELECT DISTINCT KEYWORD FROM mutation_event " + - "WHERE MUTATION_EVENT_ID in (" + concatEventIds + ")"; + String sql = "SELECT DISTINCT keyword FROM mutation_event " + + "WHERE mutation_event_id in (" + concatEventIds + ")"; pstmt = con.prepareStatement(sql); Set set = new HashSet(); rs = pstmt.executeQuery(); @@ -1510,12 +1478,10 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE mutation"); pstmt.executeUpdate(); pstmt = con.prepareStatement("TRUNCATE TABLE mutation_event"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { @@ -1547,13 +1513,13 @@ public static List getTiers(String _cancerStudyStableIds) throws DaoExce try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( - "SELECT DISTINCT DRIVER_TIERS_FILTER FROM alteration_driver_annotation " - + "WHERE DRIVER_TIERS_FILTER is not NULL AND DRIVER_TIERS_FILTER <> '' AND GENETIC_PROFILE_ID=? " - + "ORDER BY DRIVER_TIERS_FILTER"); + "SELECT DISTINCT driver_tiers_filter FROM alteration_driver_annotation " + + "WHERE driver_tiers_filter is not NULL AND driver_tiers_filter <> '' AND genetic_profile_id=? " + + "ORDER BY driver_tiers_filter"); pstmt.setLong(1, geneticProfile.getGeneticProfileId()); rs = pstmt.executeQuery(); while (rs.next()) { - tiers.add(rs.getString("DRIVER_TIERS_FILTER")); + tiers.add(rs.getString("driver_tiers_filter")); } } catch (SQLException e) { throw new DaoException(e); @@ -1603,12 +1569,12 @@ public static boolean hasDriverAnnotations(String _cancerStudyStableIds) throws try { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( - "SELECT DISTINCT DRIVER_FILTER FROM alteration_driver_annotation " - + "WHERE DRIVER_FILTER is not NULL AND DRIVER_FILTER <> '' AND GENETIC_PROFILE_ID=? "); + "SELECT DISTINCT driver_filter FROM alteration_driver_annotation " + + "WHERE driver_filter is not NULL AND driver_filter <> '' AND genetic_profile_id=? "); pstmt.setLong(1, geneticProfile.getGeneticProfileId()); rs = pstmt.executeQuery(); while (rs.next()) { - driverValues.add(rs.getString("DRIVER_FILTER")); + driverValues.add(rs.getString("driver_filter")); } } catch (SQLException e) { throw new DaoException(e); @@ -1642,8 +1608,8 @@ public static boolean hasDuplicateMutationEvents() throws DaoException { con = JdbcUtil.getDbConnection(DaoMutation.class); pstmt = con.prepareStatement( "SELECT " - + "(SELECT COUNT(*) FROM mutation_event) = " - + "(SELECT COUNT(DISTINCT ENTREZ_GENE_ID, CHR, START_POSITION, END_POSITION, TUMOR_SEQ_ALLELE, PROTEIN_CHANGE, MUTATION_TYPE) FROM mutation_event)"); + + "(SELECT count(*) FROM mutation_event) = " + + "(SELECT count(DISTINCT entrez_gene_id, chr, start_position, end_position, tumor_seq_allele, protein_change, mutation_type) FROM mutation_event)"); rs = pstmt.executeQuery(); if (rs.next()) { sqlQueryResult = rs.getInt(1); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoPatient.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoPatient.java index b143a3e9..e2119bed 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoPatient.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoPatient.java @@ -111,30 +111,39 @@ public static void cachePatient(Patient patient, int cancerStudyId) } } + private static final String PATIENT_SEQUENCE = "seq_patient"; + public static int addPatient(Patient patient) throws DaoException { Connection con = null; PreparedStatement pstmt = null; - ResultSet rs = null; try { - con = JdbcUtil.getDbConnection(DaoPatient.class); - pstmt = con.prepareStatement("INSERT INTO patient (`STABLE_ID`, `CANCER_STUDY_ID`) VALUES (?,?)", - Statement.RETURN_GENERATED_KEYS); - pstmt.setString(1, patient.getStableId()); - pstmt.setInt(2, patient.getCancerStudy().getInternalId()); - pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - cachePatient(new Patient(patient.getCancerStudy(), patient.getStableId(), rs.getInt(1)), patient.getCancerStudy().getInternalId()); - return rs.getInt(1); + long internalId = ClickHouseAutoIncrement.nextId(PATIENT_SEQUENCE); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader loader = ClickHouseBulkLoader.getClickHouseBulkLoader("patient"); + loader.setFieldNames(new String[]{"internal_id", "stable_id", "cancer_study_id"}); + loader.insertRecord( + Long.toString(internalId), + patient.getStableId(), + Integer.toString(patient.getCancerStudy().getInternalId()) + ); + } else { + con = JdbcUtil.getDbConnection(DaoPatient.class); + pstmt = con.prepareStatement("INSERT INTO patient (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (?,?,?)"); + pstmt.setLong(1, internalId); + pstmt.setString(2, patient.getStableId()); + pstmt.setInt(3, patient.getCancerStudy().getInternalId()); + pstmt.executeUpdate(); } - return -1; + cachePatient(new Patient(patient.getCancerStudy(), patient.getStableId(), (int) internalId), + patient.getCancerStudy().getInternalId()); + return (int) internalId; } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoPatient.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoPatient.class, con, pstmt, null); } } @@ -166,10 +175,8 @@ public static void deleteAllRecords() throws DaoException ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoPatient.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE patient"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); @@ -184,38 +191,40 @@ public static void deleteAllRecords() throws DaoException public static void createSampleCountClinicalData(int cancerStudyId) throws DaoException { Connection con = null; PreparedStatement pstmt = null; - ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoCopyNumberSegment.class); pstmt = con.prepareStatement( - "REPLACE `clinical_patient`" + - "SELECT patient.`INTERNAL_ID`, 'SAMPLE_COUNT', COUNT(*) FROM sample " + - "INNER JOIN patient ON sample.`PATIENT_ID` = patient.`INTERNAL_ID`" + - "WHERE patient.`CANCER_STUDY_ID`=? " + - "GROUP BY patient.`INTERNAL_ID`;"); - pstmt.setInt(1, cancerStudyId); + "INSERT INTO clinical_patient(`internal_id`, `attr_id`, `attr_value`) " + + "SELECT patient.`internal_id` AS internal_id, ? as attr_id, CAST(COUNT(*) AS CHAR) AS sample_count FROM sample " + + "INNER JOIN patient ON sample.`patient_id` = patient.`internal_id` " + + "WHERE patient.`cancer_study_id`=? " + + "GROUP BY patient.`internal_id`"); + pstmt.setString(1, SAMPLE_COUNT_ATTR_ID); + pstmt.setInt(2, cancerStudyId); + pstmt.executeUpdate(); ClinicalAttribute clinicalAttribute = DaoClinicalAttributeMeta.getDatum(SAMPLE_COUNT_ATTR_ID, cancerStudyId); if (clinicalAttribute == null) { ClinicalAttribute attr = new ClinicalAttribute(SAMPLE_COUNT_ATTR_ID, "Number of Samples Per Patient", "Number of Samples Per Patient", "STRING", true, "1", cancerStudyId); DaoClinicalAttributeMeta.addDatum(attr); } - pstmt.executeUpdate(); + // Forces ClickHouse to finalize the data insertion causing removing of duplicates in ReplacingMergeTree engine. Expensive operation. + con.prepareStatement("OPTIMIZE TABLE clinical_patient FINAL").executeUpdate(); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoCopyNumberSegment.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoCopyNumberSegment.class, con, pstmt, null); } } private static Patient extractPatient(ResultSet rs) throws SQLException { try { - CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByInternalId(rs.getInt("CANCER_STUDY_ID")); + CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByInternalId(rs.getInt("cancer_study_id")); if (cancerStudy == null) return null; return new Patient(cancerStudy, - rs.getString("STABLE_ID"), - rs.getInt("INTERNAL_ID")); + rs.getString("stable_id"), + rs.getInt("internal_id")); } catch (DaoException e) { throw new SQLException(e); @@ -243,23 +252,21 @@ public static void deletePatients(int internalStudyId, Set patientStable DaoSample.deleteSamples(internalStudyId, patientsSampleStableIds); Connection con = null; - PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoPatient.class); - pstmt = con.prepareStatement("DELETE FROM `patient` WHERE `INTERNAL_ID` IN (" - + String.join(",", Collections.nCopies(internalPatientIds.size(), "?")) - + ")"); - int parameterIndex = 1; - for (Integer internalPatientId : internalPatientIds) { - pstmt.setInt(parameterIndex++, internalPatientId); - }; - pstmt.executeUpdate(); + List clinicalEventIds = collectIds(con, + "SELECT clinical_event_id FROM clinical_event WHERE patient_id IN ", internalPatientIds); + deleteByIds(con, "DELETE FROM clinical_event_data WHERE clinical_event_id IN ", clinicalEventIds); + deleteByIds(con, "DELETE FROM clinical_event WHERE clinical_event_id IN ", clinicalEventIds); + deleteByIds(con, "DELETE FROM clinical_patient WHERE internal_id IN ", internalPatientIds); + deleteByIds(con, "DELETE FROM resource_patient WHERE internal_id IN ", internalPatientIds); + deleteByIds(con, "DELETE FROM patient WHERE internal_id IN ", internalPatientIds); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoPatient.class, con, pstmt, null); + JdbcUtil.closeAll(DaoPatient.class, con, null, null); } log.info("Removing {} patients from study with internal id={} done.", patientStableIds, internalStudyId); } @@ -275,4 +282,42 @@ public static Set findInternalPatientIdsInStudy(Integer internalStudyId } return internalPatientIds; } + + private static List collectIds(Connection con, String sqlPrefix, Collection ids) throws SQLException { + if (ids == null || ids.isEmpty()) { + return Collections.emptyList(); + } + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + PreparedStatement pstmt = null; + ResultSet rs = null; + List collected = new ArrayList<>(); + try { + pstmt = con.prepareStatement(sqlPrefix + "(" + placeholders + ")"); + int index = 1; + for (Integer id : ids) { + pstmt.setInt(index++, id); + } + rs = pstmt.executeQuery(); + while (rs.next()) { + collected.add(rs.getInt(1)); + } + } finally { + JdbcUtil.closeAll(DaoPatient.class, null, pstmt, rs); + } + return collected; + } + + private static void deleteByIds(Connection con, String sqlPrefix, Collection ids) throws SQLException { + if (ids == null || ids.isEmpty()) { + return; + } + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + try (PreparedStatement pstmt = con.prepareStatement(sqlPrefix + "(" + placeholders + ")")) { + int index = 1; + for (Integer id : ids) { + pstmt.setInt(index++, id); + } + pstmt.executeUpdate(); + } + } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenome.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenome.java index da829fad..78f72efa 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenome.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenome.java @@ -27,6 +27,8 @@ */ public final class DaoReferenceGenome { + private static final String REFERENCE_GENOME_SEQUENCE = "seq_reference_genome"; + private static final Map byGenomeBuild = new HashMap(); private static final Map byGenomeName = new HashMap(); private static final Map byGenomeInternalId = new HashMap(); @@ -100,26 +102,22 @@ public static void addReferenceGenome(ReferenceGenome referenceGenome, boolean o else { Connection con = null; PreparedStatement pstmt = null; - ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoReferenceGenome.class); + long refId = ClickHouseAutoIncrement.nextId(REFERENCE_GENOME_SEQUENCE); pstmt = con.prepareStatement("INSERT INTO reference_genome " + - "( `species`, `name`, " - + "`build_name`, `genome_size`, `URL`, " - + " `release_date` ) VALUES (?,?,?,?,?,?)", - Statement.RETURN_GENERATED_KEYS); - pstmt.setString(1, referenceGenome.getSpecies()); - pstmt.setString(2, referenceGenome.getGenomeName()); - pstmt.setString(3, referenceGenome.getBuildName()); - pstmt.setLong(4, referenceGenome.getGenomeSize()); - pstmt.setString(5, referenceGenome.getUrl()); - pstmt.setDate(6, new java.sql.Date(referenceGenome.getReleaseDate().getTime())); + "( `reference_genome_id`, `species`, `name`, " + + "`build_name`, `genome_size`, `url`, " + + " `release_date` ) VALUES (?,?,?,?,?,?,?)"); + pstmt.setLong(1, refId); + pstmt.setString(2, referenceGenome.getSpecies()); + pstmt.setString(3, referenceGenome.getGenomeName()); + pstmt.setString(4, referenceGenome.getBuildName()); + pstmt.setLong(5, referenceGenome.getGenomeSize()); + pstmt.setString(6, referenceGenome.getUrl()); + pstmt.setDate(7, new java.sql.Date(referenceGenome.getReleaseDate().getTime())); if (pstmt.executeUpdate() != 0 ) { - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - int autoId = rs.getInt(1); - referenceGenome.setReferenceGenomeId(autoId); - } + referenceGenome.setReferenceGenomeId((int) refId); // update reference cache addCache(referenceGenome); } else { @@ -128,7 +126,7 @@ public static void addReferenceGenome(ReferenceGenome referenceGenome, boolean o } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoReferenceGenome.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoReferenceGenome.class, con, pstmt, null); } } } @@ -143,10 +141,8 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoReferenceGenome.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE reference_genome"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); // clear cache clearCache(); } catch (SQLException e) { @@ -174,7 +170,7 @@ public static void updateReferenceGenome(ReferenceGenome referenceGenome) throws con = JdbcUtil.getDbConnection(DaoReferenceGenome.class); pstmt = con.prepareStatement("UPDATE reference_genome " + "SET `species`=?, `name`=?, " - + "`build_name`=?, `genome_size`=?, `URL`=?, " + + "`build_name`=?, `genome_size`=?, `url`=?, " + " `release_date`=? WHERE `reference_genome_id`=?"); pstmt.setString(1, referenceGenome.getSpecies()); pstmt.setString(2, referenceGenome.getGenomeName()); @@ -245,13 +241,13 @@ public static int getReferenceGenomeIdByName(String name) throws DaoException { */ private static ReferenceGenome extractReferenceGenome(ResultSet rs) throws SQLException { ReferenceGenome referenceGenome = new ReferenceGenome( - rs.getString("NAME"), - rs.getString("SPECIES"), - rs.getString("BUILD_NAME")); - referenceGenome.setReferenceGenomeId(rs.getInt("REFERENCE_GENOME_ID")); - referenceGenome.setGenomeSize(rs.getLong("GENOME_SIZE")); - referenceGenome.setReleaseDate(rs.getDate("RELEASE_DATE")); - referenceGenome.setUrl(rs.getString("URL")); + rs.getString("name"), + rs.getString("species"), + rs.getString("build_name")); + referenceGenome.setReferenceGenomeId(rs.getInt("reference_genome_id")); + referenceGenome.setGenomeSize(rs.getLong("genome_size")); + referenceGenome.setReleaseDate(rs.getDate("release_date")); + referenceGenome.setUrl(rs.getString("url")); return referenceGenome; } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenomeGene.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenomeGene.java index b4361930..29f68d69 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenomeGene.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoReferenceGenomeGene.java @@ -44,10 +44,10 @@ public void updateGene(ReferenceGenomeGene gene) throws DaoException { PreparedStatement pstmt = null; ResultSet rs = null; try { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); con = JdbcUtil.getDbConnection(DaoReferenceGenome.class); pstmt = con.prepareStatement - ("UPDATE reference_genome_gene SET `START`=?, `END`=? WHERE `ENTREZ_GENE_ID`=? AND `REFERENCE_GENOME_ID`=?"); + ("UPDATE reference_genome_gene SET `start`=?, `end`=? WHERE `entrez_gene_id`=? AND `reference_genome_id`=?"); pstmt.setLong(1, gene.getStart()); pstmt.setLong(2, gene.getEnd()); pstmt.setLong(3, gene.getEntrezGeneId()); @@ -56,7 +56,7 @@ public void updateGene(ReferenceGenomeGene gene) throws DaoException { } catch (SQLException e) { throw new DaoException(e); } finally { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); JdbcUtil.closeAll(DaoGene.class, con, pstmt, rs); } } @@ -77,7 +77,7 @@ public void addOrUpdateGene(ReferenceGenomeGene gene) throws DaoException { //add gene, referring to this genetic entity con = JdbcUtil.getDbConnection(DaoGene.class); pstmt = con.prepareStatement - ("INSERT INTO `reference_genome_gene` (`ENTREZ_GENE_ID`, `REFERENCE_GENOME_ID`,`CHR`,`CYTOBAND`,`START`,`END`) " + ("INSERT INTO `reference_genome_gene` (`entrez_gene_id`, `reference_genome_id`,`chr`,`cytoband`,`start`,`end`) " + "VALUES (?,?,?,?,?,?)"); pstmt.setLong(1, gene.getEntrezGeneId()); pstmt.setInt(2, gene.getReferenceGenomeId()); @@ -111,7 +111,7 @@ public ReferenceGenomeGene getGene(long entrezGeneId, int referenceGenomeId) thr try { con = JdbcUtil.getDbConnection(DaoReferenceGenomeGene.class); pstmt = con.prepareStatement - ("SELECT * FROM `reference_genome_gene` WHERE `ENTREZ_GENE_ID` = ? AND `REFERENCE_GENOME_ID` = ?"); + ("SELECT * FROM `reference_genome_gene` WHERE `entrez_gene_id` = ? AND `reference_genome_id` = ?"); pstmt.setLong(1, entrezGeneId); pstmt.setInt(2, referenceGenomeId); rs = pstmt.executeQuery(); @@ -128,14 +128,14 @@ public ReferenceGenomeGene getGene(long entrezGeneId, int referenceGenomeId) thr } private ReferenceGenomeGene extractGene(ResultSet rs) throws SQLException, DaoException { - int entrezGeneId = rs.getInt("ENTREZ_GENE_ID"); - int reference_genome_id = rs.getInt("REFERENCE_GENOME_ID"); - String cytoband = rs.getString("CYTOBAND"); + int entrezGeneId = rs.getInt("entrez_gene_id"); + int reference_genome_id = rs.getInt("reference_genome_id"); + String cytoband = rs.getString("cytoband"); ReferenceGenomeGene gene = new ReferenceGenomeGene(entrezGeneId, reference_genome_id); - gene.setChr(rs.getString("CHR")); - gene.setCytoband(rs.getString("CYTOBAND")); - gene.setStart(rs.getLong("START")); - gene.setEnd(rs.getLong("END")); + gene.setChr(rs.getString("chr")); + gene.setCytoband(rs.getString("cytoband")); + gene.setStart(rs.getLong("start")); + gene.setEnd(rs.getLong("end")); return gene; } @@ -150,7 +150,7 @@ public void deleteGene(int entrezGeneId, int referenceGenomeId) throws DaoExcept ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoReferenceGenomeGene.class); - pstmt = con.prepareStatement("DELETE FROM `reference_genome_gene` WHERE ENTREZ_GENE_ID=? AND REFERENCE_GENOME_ID=?"); + pstmt = con.prepareStatement("DELETE FROM `reference_genome_gene` WHERE entrez_gene_id=? AND reference_genome_id=?"); pstmt.setLong(1, entrezGeneId); pstmt.setInt(2, referenceGenomeId); pstmt.executeUpdate(); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceData.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceData.java index 34063340..03edb8a5 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceData.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceData.java @@ -6,40 +6,43 @@ import org.mskcc.cbio.portal.model.ResourceBaseData; /** - * Data Access Object for `resource` tables + * Data Access Object for `resource_node` table */ public final class DaoResourceData { - public static final String RESOURCE_SAMPLE_TABLE = "resource_sample"; - public static final String RESOURCE_PATIENT_TABLE = "resource_patient"; - public static final String RESOURCE_STUDY_TABLE = "resource_study"; + public static final String RESOURCE_NODE_TABLE = "resource_node"; - private static final String SAMPLE_INSERT = "INSERT INTO " + RESOURCE_SAMPLE_TABLE - + "(`INTERNAL_ID`,`RESOURCE_ID`,`URL` VALUES(?,?,?)"; - private static final String PATIENT_INSERT = "INSERT INTO " + RESOURCE_PATIENT_TABLE - + "(`INTERNAL_ID`,`RESOURCE_ID`,`URL` VALUES(?,?,?)"; - private static final String STUDY_INSERT = "INSERT INTO " + RESOURCE_STUDY_TABLE - + "(`INTERNAL_ID`,`RESOURCE_ID`,`URL` VALUES(?,?,?)"; + private static final String INSERT = "INSERT INTO " + RESOURCE_NODE_TABLE + + "(`resource_id`,`cancer_study_id`,`entity_type`,`entity_internal_id`,`display_name`,`url`,`metadata`,`group_path`)" + + " VALUES(?,?,?,?,?,?,?,?)"; private DaoResourceData() { } - public static int addSampleDatum(int internalSampleId, String resourceId, String url) throws DaoException { - return addDatum(SAMPLE_INSERT, RESOURCE_SAMPLE_TABLE, internalSampleId, resourceId, url); + public static int addSampleDatum(int internalSampleId, int cancerStudyId, String resourceId, String displayName, String url, String metadata, String groupPath) throws DaoException { + return addDatum(internalSampleId, cancerStudyId, "SAMPLE", resourceId, displayName, url, metadata, groupPath); } - public static int addPatientDatum(int internalPatientId, String resourceId, String url) throws DaoException { - return addDatum(PATIENT_INSERT, RESOURCE_PATIENT_TABLE, internalPatientId, resourceId, url); + public static int addPatientDatum(int internalPatientId, int cancerStudyId, String resourceId, String displayName, String url, String metadata, String groupPath) throws DaoException { + return addDatum(internalPatientId, cancerStudyId, "PATIENT", resourceId, displayName, url, metadata, groupPath); } - public static int addStudyDatum(int internalStudyId, String resourceId, String url) throws DaoException { - return addDatum(STUDY_INSERT, RESOURCE_STUDY_TABLE, internalStudyId, resourceId, url); + public static int addStudyDatum(int internalStudyId, int cancerStudyId, String resourceId, String displayName, String url, String metadata, String groupPath) throws DaoException { + return addDatum(internalStudyId, cancerStudyId, "STUDY", resourceId, displayName, url, metadata, groupPath); } - public static int addDatum(String query, String tableName, int internalId, String resourceId, String url) - throws DaoException { - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.getMySQLbulkLoader(tableName).insertRecord(Integer.toString(internalId), resourceId, url); + public static int addDatum(int internalId, int cancerStudyId, String entityType, + String resourceId, String displayName, String url, String metadata, String groupPath) throws DaoException { + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.getClickHouseBulkLoader(RESOURCE_NODE_TABLE).insertRecord( + resourceId, + Integer.toString(cancerStudyId), + entityType, + Integer.toString(internalId), + displayName, + url, + metadata, + groupPath); return 1; } @@ -48,12 +51,23 @@ public static int addDatum(String query, String tableName, int internalId, Strin ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoResourceData.class); - - pstmt = con.prepareStatement(query); - pstmt.setInt(1, internalId); - pstmt.setString(2, resourceId); - pstmt.setString(3, url); - + pstmt = con.prepareStatement(INSERT); + pstmt.setString(1, resourceId); + pstmt.setInt(2, cancerStudyId); + pstmt.setString(3, entityType); + pstmt.setInt(4, internalId); + pstmt.setString(5, displayName); + pstmt.setString(6, url); + if (metadata != null) { + pstmt.setString(7, metadata); + } else { + pstmt.setNull(7, Types.VARCHAR); + } + if (groupPath != null) { + pstmt.setString(8, groupPath); + } else { + pstmt.setNull(8, Types.VARCHAR); + } return pstmt.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); @@ -62,35 +76,32 @@ public static int addDatum(String query, String tableName, int internalId, Strin } } - public static List getDataByPatientId(int cancerStudyId, String patientId) throws DaoException - { + public static List getDataByPatientId(int cancerStudyId, String patientId) throws DaoException { List internalIds = new ArrayList(); internalIds.add(DaoPatient.getPatientByCancerStudyAndPatientId(cancerStudyId, patientId).getInternalId()); - return getDataByInternalIds(cancerStudyId, RESOURCE_PATIENT_TABLE, internalIds); + return getDataByInternalIds(cancerStudyId, "PATIENT", internalIds); } - private static List getDataByInternalIds(int internalCancerStudyId, String table, List internalIds) throws DaoException - { + private static List getDataByInternalIds(int internalCancerStudyId, String entityType, List internalIds) throws DaoException { Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; List resources = new ArrayList(); - String sql = ("SELECT * FROM " + table + " WHERE `INTERNAL_ID` IN " + - "(" + generateIdsSql(internalIds) + ")"); + String sql = "SELECT * FROM " + RESOURCE_NODE_TABLE + + " WHERE `entity_type` = '" + entityType + "'" + + " AND `entity_internal_id` IN (" + generateIdsSql(internalIds) + ")"; try { con = JdbcUtil.getDbConnection(DaoResourceData.class); pstmt = con.prepareStatement(sql); rs = pstmt.executeQuery(); while (rs.next()) { - resources.add(extract(table, internalCancerStudyId, rs)); + resources.add(extract(internalCancerStudyId, entityType, rs)); } - } - catch (SQLException e) { + } catch (SQLException e) { throw new DaoException(e); - } - finally { + } finally { JdbcUtil.closeAll(DaoResourceData.class, con, pstmt, rs); } @@ -101,13 +112,20 @@ private static String generateIdsSql(Collection ids) { return "'" + StringUtils.join(ids, "','") + "'"; } - private static ResourceBaseData extract(String table, int internalCancerStudyId, ResultSet rs) throws SQLException { - String stableId = getStableIdFromInternalId(table, rs.getInt("INTERNAL_ID")); - return new ResourceBaseData(internalCancerStudyId, stableId, rs.getString("RESOURCE_ID"), rs.getString("URL")); + private static ResourceBaseData extract(int internalCancerStudyId, String entityType, ResultSet rs) throws SQLException { + int internalId = rs.getInt("entity_internal_id"); + String stableId = getStableIdFromInternalId(entityType, internalId); + return new ResourceBaseData( + internalCancerStudyId, + stableId, + rs.getString("resource_id"), + rs.getString("url"), + rs.getString("metadata"), + rs.getString("group_path")); } - private static String getStableIdFromInternalId(String table, int internalId) { - if (table.equals(RESOURCE_SAMPLE_TABLE)) { + private static String getStableIdFromInternalId(String entityType, int internalId) { + if ("SAMPLE".equals(entityType)) { return DaoSample.getSampleById(internalId).getStableId(); } else { return DaoPatient.getPatientById(internalId).getStableId(); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceDefinition.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceDefinition.java index 8de9f5bc..1c30b4b3 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceDefinition.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoResourceDefinition.java @@ -17,8 +17,8 @@ public static int addDatum(ResourceDefinition resource) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoResourceDefinition.class); - pstmt = con.prepareStatement("INSERT INTO resource_definition(" + "`RESOURCE_ID`," + "`DISPLAY_NAME`," - + "`DESCRIPTION`," + "`RESOURCE_TYPE`," + "`OPEN_BY_DEFAULT`," + "`PRIORITY`," + "`CANCER_STUDY_ID`," + "`CUSTOM_METADATA`)" + pstmt = con.prepareStatement("INSERT INTO resource_definition(" + "`resource_id`," + "`display_name`," + + "`description`," + "`resource_type`," + "`open_by_default`," + "`priority`," + "`cancer_study_id`," + "`custom_metadata`)" + " VALUES(?,?,?,?,?,?,?,?)"); pstmt.setString(1, resource.getResourceId()); pstmt.setString(2, resource.getDisplayName()); @@ -37,9 +37,9 @@ public static int addDatum(ResourceDefinition resource) throws DaoException { } private static ResourceDefinition unpack(ResultSet rs) throws SQLException { - return new ResourceDefinition(rs.getString("RESOURCE_ID"), rs.getString("DISPLAY_NAME"), rs.getString("DESCRIPTION"), - ResourceType.valueOf(rs.getString("RESOURCE_TYPE")), rs.getBoolean("OPEN_BY_DEFAULT"), rs.getInt("PRIORITY"), - rs.getInt("CANCER_STUDY_ID"), rs.getString("CUSTOM_METADATA")); + return new ResourceDefinition(rs.getString("resource_id"), rs.getString("display_name"), rs.getString("description"), + ResourceType.valueOf(rs.getString("resource_type")), rs.getBoolean("open_by_default"), rs.getInt("priority"), + rs.getInt("cancer_study_id"), rs.getString("custom_metadata")); } public static ResourceDefinition getDatum(String resourceId, Integer cancerStudyId) throws DaoException { @@ -62,8 +62,8 @@ public static List getDatum(Collection resourceIds, try { con = JdbcUtil.getDbConnection(DaoResourceDefinition.class); - pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE RESOURCE_ID IN ('" - + StringUtils.join(resourceIds, "','") + "') AND CANCER_STUDY_ID=" + String.valueOf(cancerStudyId)); + pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE resource_id IN ('" + + StringUtils.join(resourceIds, "','") + "') AND cancer_study_id=" + String.valueOf(cancerStudyId)); rs = pstmt.executeQuery(); @@ -90,7 +90,7 @@ public static List getDatum(Collection resourceIds) try { con = JdbcUtil.getDbConnection(DaoResourceDefinition.class); - pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE RESOURCE_ID IN ('" + pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE resource_id IN ('" + StringUtils.join(resourceIds, "','") + "')"); rs = pstmt.executeQuery(); @@ -115,7 +115,7 @@ public static List getDatumByStudy(int cancerStudyId) throws try { con = JdbcUtil.getDbConnection(DaoResourceDefinition.class); - pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE CANCER_STUDY_ID=" + String.valueOf(cancerStudyId)); + pstmt = con.prepareStatement("SELECT * FROM resource_definition WHERE cancer_study_id=" + String.valueOf(cancerStudyId)); rs = pstmt.executeQuery(); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoSample.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoSample.java index 7f1434a4..59da75af 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoSample.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoSample.java @@ -133,34 +133,44 @@ private static void cacheSample(Sample sample, int cancerStudyId) samples.put(sample.getStableId(), sample); } + private static final String SAMPLE_SEQUENCE = "seq_sample"; + public static int addSample(Sample sample) throws DaoException { Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; try { - con = JdbcUtil.getDbConnection(DaoSample.class); - pstmt = con.prepareStatement("INSERT INTO sample " + - "( `STABLE_ID`, `SAMPLE_TYPE`, `PATIENT_ID` ) " + - "VALUES (?,?,?)", - Statement.RETURN_GENERATED_KEYS); - pstmt.setString(1, sample.getStableId()); - pstmt.setString(2, sample.getType().toString()); - pstmt.setInt(3, sample.getInternalPatientId()); - pstmt.executeUpdate(); - rs = pstmt.getGeneratedKeys(); - if (rs.next()) { - cacheSample(new Sample(rs.getInt(1), sample.getStableId(), - sample.getInternalPatientId())); - return rs.getInt(1); + long internalId = ClickHouseAutoIncrement.nextId(SAMPLE_SEQUENCE); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader loader = ClickHouseBulkLoader.getClickHouseBulkLoader("sample"); + loader.setFieldNames(new String[]{"internal_id", "stable_id", "sample_type", "patient_id"}); + loader.insertRecord( + Long.toString(internalId), + sample.getStableId(), + sample.getType().toString(), + Integer.toString(sample.getInternalPatientId()) + ); + } else { + con = JdbcUtil.getDbConnection(DaoSample.class); + pstmt = con.prepareStatement("INSERT INTO sample " + + "( `internal_id`, `stable_id`, `sample_type`, `patient_id` ) " + + "VALUES (?,?,?,?)"); + pstmt.setLong(1, internalId); + pstmt.setString(2, sample.getStableId()); + pstmt.setString(3, sample.getType().toString()); + pstmt.setInt(4, sample.getInternalPatientId()); + pstmt.executeUpdate(); } - return -1; + cacheSample(new Sample((int) internalId, sample.getStableId(), + sample.getInternalPatientId())); + return (int) internalId; } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoSample.class, con, pstmt, rs); + JdbcUtil.closeAll(DaoSample.class, con, pstmt, null); } } @@ -255,10 +265,8 @@ public static void deleteAllRecords() throws DaoException PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoSample.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE sample"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); @@ -289,32 +297,35 @@ public static void deleteSamples(int internalStudyId, Set sampleStableId removeSamplesInGeneticAlterationsForStudy(internalStudyId, internalSampleIds); Connection con = null; - PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoSample.class); - pstmt = con.prepareStatement("DELETE FROM `sample` WHERE `INTERNAL_ID` IN (" - + String.join(",", Collections.nCopies(internalSampleIds.size(), "?")) - + ")"); - int parameterIndex = 1; - for (Integer internalSampleId : internalSampleIds) { - pstmt.setInt(parameterIndex++, internalSampleId); - }; - pstmt.executeUpdate(); + deleteByIds(con, "DELETE FROM alteration_driver_annotation WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM sample_cna_event WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM mutation WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM structural_variant WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM sample_profile WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM sample_list_list WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM copy_number_seg WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM allele_specific_copy_number WHERE sample_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM clinical_sample WHERE internal_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM resource_sample WHERE internal_id IN ", internalSampleIds); + deleteByIds(con, "DELETE FROM sample WHERE internal_id IN ", internalSampleIds); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoSample.class, con, pstmt, null); + JdbcUtil.closeAll(DaoSample.class, con, null, null); } + DaoCancerStudy.purgeUnreferencedRecordsAfterDeletionOfStudy(); log.info("Removing {} samples from study with internal id={} done.", sampleStableIds, internalStudyId); } private static Sample extractSample(ResultSet rs) throws SQLException { - return new Sample(rs.getInt("INTERNAL_ID"), - rs.getString("STABLE_ID"), - rs.getInt("PATIENT_ID")); + return new Sample(rs.getInt("internal_id"), + rs.getString("stable_id"), + rs.getInt("patient_id")); } /** @@ -383,4 +394,18 @@ public static Set findInternalSampleIdsInStudy(Integer internalStudyId, } return internalSampleIds; } + + private static void deleteByIds(Connection con, String sqlPrefix, Collection ids) throws SQLException { + if (ids == null || ids.isEmpty()) { + return; + } + String placeholders = String.join(",", Collections.nCopies(ids.size(), "?")); + try (PreparedStatement pstmt = con.prepareStatement(sqlPrefix + "(" + placeholders + ")")) { + int index = 1; + for (Integer id : ids) { + pstmt.setInt(index++, id); + } + pstmt.executeUpdate(); + } + } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleList.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleList.java index f3fd4ab4..fb4e1c49 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleList.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleList.java @@ -43,7 +43,9 @@ */ public class DaoSampleList { - private static final String DELETE_SAMPLE_LIST_LIST = "DELETE FROM sample_list_list WHERE `LIST_ID` = ?"; + private static final String DELETE_SAMPLE_LIST_LIST = "DELETE FROM sample_list_list WHERE `list_id` = ?"; + + private static final String SAMPLE_LIST_SEQUENCE = "seq_sample_list"; /** * Adds record to sample_list table. @@ -55,22 +57,17 @@ public int addSampleList(SampleList sampleList) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoSampleList.class); - pstmt = con.prepareStatement("INSERT INTO sample_list (`STABLE_ID`, `CANCER_STUDY_ID`, `NAME`, `CATEGORY`," + - "`DESCRIPTION`)" + " VALUES (?,?,?,?,?)", Statement.RETURN_GENERATED_KEYS); - pstmt.setString(1, sampleList.getStableId()); - pstmt.setInt(2, sampleList.getCancerStudyId()); - pstmt.setString(3, sampleList.getName()); - pstmt.setString(4, sampleList.getSampleListCategory().getCategory()); - pstmt.setString(5, sampleList.getDescription()); + long listId = ClickHouseAutoIncrement.nextId(SAMPLE_LIST_SEQUENCE); + pstmt = con.prepareStatement("INSERT INTO sample_list (`list_id`, `stable_id`, `cancer_study_id`, `name`, `category`," + + "`description`)" + " VALUES (?,?,?,?,?,?)"); + pstmt.setLong(1, listId); + pstmt.setString(2, sampleList.getStableId()); + pstmt.setInt(3, sampleList.getCancerStudyId()); + pstmt.setString(4, sampleList.getName()); + pstmt.setString(5, sampleList.getSampleListCategory().getCategory()); + pstmt.setString(6, sampleList.getDescription()); rows = pstmt.executeUpdate(); - try (ResultSet generatedKey = pstmt.getGeneratedKeys()) { - if (generatedKey.next()) { - int listId = generatedKey.getInt(1); - addSampleListList(sampleList.getCancerStudyId(), listId, sampleList.getSampleList(), con); - } else { - throw new DaoException("Creating sample list failed, no ID obtained."); - } - } + addSampleListList(sampleList.getCancerStudyId(), (int) listId, sampleList.getSampleList(), con); } catch (SQLException e) { throw new DaoException(e); } finally { @@ -90,7 +87,7 @@ public SampleList getSampleListByStableId(String stableId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoSampleList.class); pstmt = con.prepareStatement - ("SELECT * FROM sample_list WHERE STABLE_ID = ?"); + ("SELECT * FROM sample_list WHERE stable_id = ?"); pstmt.setString(1, stableId); rs = pstmt.executeQuery(); if (rs.next()) { @@ -116,7 +113,7 @@ public SampleList getSampleListById(int id) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoSampleList.class); pstmt = con.prepareStatement - ("SELECT * FROM sample_list WHERE LIST_ID = ?"); + ("SELECT * FROM sample_list WHERE list_id = ?"); pstmt.setInt(1, id); rs = pstmt.executeQuery(); if (rs.next()) { @@ -143,7 +140,7 @@ public ArrayList getAllSampleLists( int cancerStudyId) throws DaoExc con = JdbcUtil.getDbConnection(DaoSampleList.class); pstmt = con.prepareStatement - ("SELECT * FROM sample_list WHERE CANCER_STUDY_ID = ? ORDER BY NAME"); + ("SELECT * FROM sample_list WHERE cancer_study_id = ? ORDER BY name"); pstmt.setInt(1, cancerStudyId); rs = pstmt.executeQuery(); ArrayList list = new ArrayList(); @@ -225,7 +222,7 @@ private int addSampleListList(int cancerStudyId, int sampleListId, List ResultSet rs = null; int skippedPatients = 0; try { - StringBuilder sql = new StringBuilder("INSERT INTO sample_list_list (`LIST_ID`, `SAMPLE_ID`) VALUES "); + StringBuilder sql = new StringBuilder("INSERT INTO sample_list_list (`list_id`, `sample_id`) VALUES "); // NOTE - as of 12/12/14, patient lists contain sample ids for (String sampleId : sampleList) { Sample sample = DaoSample.getSampleByCancerStudyAndSampleId(cancerStudyId, sampleId); @@ -276,13 +273,13 @@ private ArrayList getSampleListList(SampleList sampleList, Connection co ResultSet rs = null; try { pstmt = con.prepareStatement - ("SELECT * FROM sample_list_list WHERE LIST_ID = ?"); + ("SELECT * FROM sample_list_list WHERE list_id = ?"); pstmt.setInt(1, sampleList.getSampleListId()); rs = pstmt.executeQuery(); ArrayList patientIds = new ArrayList(); while (rs.next()) { // NOTE - as of 12/12/14, patient lists contain sample ids - int sample_id = rs.getInt("SAMPLE_ID"); + int sample_id = rs.getInt("sample_id"); Sample sample = DaoSample.getSampleById(sample_id); patientIds.add(sample.getStableId()); } @@ -299,12 +296,12 @@ private ArrayList getSampleListList(SampleList sampleList, Connection co */ private SampleList extractSampleList(ResultSet rs) throws SQLException { SampleList sampleList = new SampleList(); - sampleList.setStableId(rs.getString("STABLE_ID")); - sampleList.setCancerStudyId(rs.getInt("CANCER_STUDY_ID")); - sampleList.setName(rs.getString("NAME")); - sampleList.setSampleListCategory(SampleListCategory.get(rs.getString("CATEGORY"))); - sampleList.setDescription(rs.getString("DESCRIPTION")); - sampleList.setSampleListId(rs.getInt("LIST_ID")); + sampleList.setStableId(rs.getString("stable_id")); + sampleList.setCancerStudyId(rs.getInt("cancer_study_id")); + sampleList.setName(rs.getString("name")); + sampleList.setSampleListCategory(SampleListCategory.get(rs.getString("category"))); + sampleList.setDescription(rs.getString("description")); + sampleList.setSampleListId(rs.getInt("list_id")); return sampleList; } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleProfile.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleProfile.java index a0a39392..d1dc85b8 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleProfile.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoSampleProfile.java @@ -32,10 +32,17 @@ package org.mskcc.cbio.portal.dao; -import java.sql.*; -import java.util.*; import org.mskcc.cbio.portal.model.Sample; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.NoSuchElementException; + /** * Data access object for sample_profile table */ @@ -47,6 +54,7 @@ public final class DaoSampleProfile { private DaoSampleProfile() {} private static final int NO_SUCH_PROFILE_ID = -1; + private static final int UPSERT_BATCH_SIZE = 1_000; public static void upsertSampleToProfileMapping(Collection sampleIds, Integer geneticProfileId, Integer panelId) throws DaoException { upsertSampleToProfileMapping( @@ -60,33 +68,73 @@ public static void upsertSampleToProfileMapping(Collection i if (idTuples.isEmpty()) { return; } + if (ClickHouseBulkLoader.isBulkLoad()) { + upsertWithBulkLoader(idTuples); + ClickHouseBulkLoader.flushAll(); + optimizeSampleProfileTable(); + return; + } + upsertWithJdbcBatch(idTuples); + optimizeSampleProfileTable(); + } + + private static void optimizeSampleProfileTable() throws DaoException { Connection con = null; - PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); + con.prepareStatement("OPTIMIZE TABLE sample_profile FINAL").executeUpdate(); + } catch (SQLException e) { + throw new DaoException(e); + } finally { + JdbcUtil.closeAll(DaoSampleProfile.class, con, null, null); + } + } - pstmt = con.prepareStatement - ("INSERT INTO sample_profile (`SAMPLE_ID`, `GENETIC_PROFILE_ID`, `PANEL_ID`)" + - " VALUES" + - String.join(",", Collections.nCopies(idTuples.size(), " (?,?,?)")) + - " ON DUPLICATE KEY UPDATE `PANEL_ID` = VALUES(`PANEL_ID`);"); - int parameterIndex = 1; + private static void upsertWithJdbcBatch(Collection idTuples) throws DaoException { + Connection con = null; + PreparedStatement insertStmt = null; + try { + con = JdbcUtil.getDbConnection(DaoSampleProfile.class); + insertStmt = con.prepareStatement( + "INSERT INTO sample_profile (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (?,?,?)"); + + int batchCount = 0; for (SampleProfileTuple idTuple : idTuples) { - pstmt.setInt(parameterIndex++, idTuple.sampleId()); - pstmt.setInt(parameterIndex++, idTuple.geneticProfileId()); - if (idTuple.panelId() != null) { - pstmt.setInt(parameterIndex, idTuple.panelId()); - } else { - pstmt.setNull(parameterIndex, java.sql.Types.INTEGER); + bindInsert(insertStmt, idTuple); + if (++batchCount % UPSERT_BATCH_SIZE == 0) { + insertStmt.executeBatch(); } - parameterIndex++; } - pstmt.executeUpdate(); + + insertStmt.executeBatch(); } catch (SQLException e) { throw new DaoException(e); } finally { - JdbcUtil.closeAll(DaoSampleProfile.class, con, pstmt, null); + JdbcUtil.closeAll(DaoSampleProfile.class, con, insertStmt, null); + } + } + + private static void upsertWithBulkLoader(Collection idTuples) throws DaoException { + ClickHouseBulkLoader loader = ClickHouseBulkLoader.getClickHouseBulkLoader("sample_profile"); + loader.setFieldNames(new String[]{"sample_id", "genetic_profile_id", "panel_id"}); + for (SampleProfileTuple idTuple : idTuples) { + loader.insertRecord( + Integer.toString(idTuple.sampleId()), + Integer.toString(idTuple.geneticProfileId()), + idTuple.panelId() != null ? Integer.toString(idTuple.panelId()) : null + ); + } + } + + private static void bindInsert(PreparedStatement insertStmt, SampleProfileTuple idTuple) throws SQLException { + insertStmt.setInt(1, idTuple.sampleId()); + insertStmt.setInt(2, idTuple.geneticProfileId()); + if (idTuple.panelId() != null) { + insertStmt.setInt(3, idTuple.panelId()); + } else { + insertStmt.setNull(3, java.sql.Types.INTEGER); } + insertStmt.addBatch(); } public static boolean sampleExistsInGeneticProfile(int sampleId, int geneticProfileId) @@ -98,7 +146,7 @@ public static boolean sampleExistsInGeneticProfile(int sampleId, int geneticProf try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); pstmt = con.prepareStatement - ("SELECT * FROM sample_profile WHERE SAMPLE_ID = ? AND GENETIC_PROFILE_ID = ?"); + ("SELECT * FROM sample_profile WHERE sample_id = ? AND genetic_profile_id = ?"); pstmt.setInt(1, sampleId); pstmt.setInt(2, geneticProfileId); rs = pstmt.executeQuery(); @@ -121,7 +169,7 @@ public static Integer getPanelId(int sampleId, int geneticProfileId) try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); pstmt = con.prepareStatement - ("SELECT PANEL_ID FROM sample_profile WHERE SAMPLE_ID = ? AND GENETIC_PROFILE_ID = ?"); + ("SELECT panel_id FROM sample_profile WHERE sample_id = ? AND genetic_profile_id = ?"); pstmt.setInt(1, sampleId); pstmt.setInt(2, geneticProfileId); rs = pstmt.executeQuery(); @@ -132,7 +180,7 @@ public static Integer getPanelId(int sampleId, int geneticProfileId) } return panelId; } else { - throw new NoSuchElementException("No sample_profile with SAMPLE_ID=" + sampleId + " and GENETIC_PROFILE_ID=" + geneticProfileId); + throw new NoSuchElementException("No sample_profile with SAMPLE_ID=" + sampleId + " and genetic_profile_id=" + geneticProfileId); } } catch (NoSuchElementException | SQLException e) { throw new DaoException(e); @@ -148,7 +196,7 @@ public static int countSamplesInProfile(int geneticProfileId) throws DaoExceptio try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); pstmt = con.prepareStatement - ("SELECT count(*) FROM sample_profile WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT count(*) FROM sample_profile WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); if (rs.next()) { @@ -169,11 +217,11 @@ public static int getProfileIdForSample(int sampleId) throws DaoException { try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); - pstmt = con.prepareStatement("SELECT GENETIC_PROFILE_ID FROM sample_profile WHERE SAMPLE_ID = ?"); + pstmt = con.prepareStatement("SELECT genetic_profile_id FROM sample_profile WHERE sample_id = ?"); pstmt.setInt(1, sampleId); rs = pstmt.executeQuery(); if( rs.next() ) { - return rs.getInt("GENETIC_PROFILE_ID"); + return rs.getInt("genetic_profile_id"); }else{ return NO_SUCH_PROFILE_ID; } @@ -193,13 +241,13 @@ public static ArrayList getAllSampleIdsInProfile(int geneticProfileId) try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); pstmt = con.prepareStatement - ("SELECT * FROM sample_profile WHERE GENETIC_PROFILE_ID = ?"); + ("SELECT * FROM sample_profile WHERE genetic_profile_id = ?"); pstmt.setInt(1, geneticProfileId); rs = pstmt.executeQuery(); ArrayList sampleIds = new ArrayList(); while (rs.next()) { - Sample sample = DaoSample.getSampleById(rs.getInt("SAMPLE_ID")); - sampleIds.add(rs.getInt("SAMPLE_ID")); + Sample sample = DaoSample.getSampleById(rs.getInt("sample_id")); + sampleIds.add(rs.getInt("sample_id")); } return sampleIds; } catch (NullPointerException e) { @@ -222,7 +270,7 @@ public static ArrayList getAllSamples() throws DaoException { rs = pstmt.executeQuery(); ArrayList sampleIds = new ArrayList(); while (rs.next()) { - sampleIds.add(rs.getInt("SAMPLE_ID")); + sampleIds.add(rs.getInt("sample_id")); } return sampleIds; } catch (NullPointerException e) { @@ -274,7 +322,7 @@ public static boolean sampleProfileMappingExistsByPanel(Integer panelId) throws ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoSampleProfile.class); - pstmt = con.prepareStatement("select count(*) from sample_profile where PANEL_ID = ?"); + pstmt = con.prepareStatement("select count(*) from sample_profile where panel_id = ?"); pstmt.setInt(1, panelId); rs = pstmt.executeQuery(); if (rs.next()) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoStructuralVariant.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoStructuralVariant.java index b411dd29..c4a69915 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoStructuralVariant.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoStructuralVariant.java @@ -39,52 +39,52 @@ private DaoStructuralVariant() { */ public static void addStructuralVariantToBulkLoader(StructuralVariant structuralVariant) throws DaoException { - MySQLbulkLoader bl = MySQLbulkLoader.getMySQLbulkLoader("structural_variant"); + ClickHouseBulkLoader bl = ClickHouseBulkLoader.getClickHouseBulkLoader("structural_variant"); String[] fieldNames = new String[]{ - "INTERNAL_ID", - "GENETIC_PROFILE_ID", - "SAMPLE_ID", - "SITE1_ENTREZ_GENE_ID", - "SITE1_ENSEMBL_TRANSCRIPT_ID", - "SITE1_CHROMOSOME", - "SITE1_POSITION", - "SITE1_CONTIG", - "SITE1_REGION", - "SITE1_REGION_NUMBER", - "SITE1_DESCRIPTION", - "SITE2_ENTREZ_GENE_ID", - "SITE2_ENSEMBL_TRANSCRIPT_ID", - "SITE2_CHROMOSOME", - "SITE2_POSITION", - "SITE2_CONTIG", - "SITE2_REGION", - "SITE2_REGION_NUMBER", - "SITE2_DESCRIPTION", - "SITE2_EFFECT_ON_FRAME", - "NCBI_BUILD", - "DNA_SUPPORT", - "RNA_SUPPORT", - "NORMAL_READ_COUNT", - "TUMOR_READ_COUNT", - "NORMAL_VARIANT_COUNT", - "TUMOR_VARIANT_COUNT", - "NORMAL_PAIRED_END_READ_COUNT", - "TUMOR_PAIRED_END_READ_COUNT", - "NORMAL_SPLIT_READ_COUNT", - "TUMOR_SPLIT_READ_COUNT", - "ANNOTATION", - "BREAKPOINT_TYPE", - "CONNECTION_TYPE", - "EVENT_INFO", - "CLASS", - "LENGTH", - "COMMENTS", - "SV_STATUS", - "ANNOTATION_JSON" + "internal_id", + "genetic_profile_id", + "sample_id", + "site1_entrez_gene_id", + "site1_ensembl_transcript_id", + "site1_chromosome", + "site1_position", + "site1_contig", + "site1_region", + "site1_region_number", + "site1_description", + "site2_entrez_gene_id", + "site2_ensembl_transcript_id", + "site2_chromosome", + "site2_position", + "site2_contig", + "site2_region", + "site2_region_number", + "site2_description", + "site2_effect_on_frame", + "ncbi_build", + "dna_support", + "rna_support", + "normal_read_count", + "tumor_read_count", + "normal_variant_count", + "tumor_variant_count", + "normal_paired_end_read_count", + "tumor_paired_end_read_count", + "normal_split_read_count", + "tumor_split_read_count", + "annotation", + "breakpoint_type", + "connection_type", + "event_info", + "class", + "length", + "comments", + "sv_status", + "annotation_json" }; bl.setFieldNames(fieldNames); - // write to the temp file maintained by the MySQLbulkLoader + // write to the temp file maintained by the ClickHouseBulkLoader bl.insertRecord( Long.toString(structuralVariant.getInternalId()), Integer.toString(structuralVariant.getGeneticProfileId()), @@ -135,7 +135,7 @@ public static void addStructuralVariantToBulkLoader(StructuralVariant structural (structuralVariant.getDriverTiersFilter() != null && !structuralVariant.getDriverTiersFilter().isEmpty() && !structuralVariant.getDriverTiersFilter().toLowerCase().equals("na"))) { - MySQLbulkLoader.getMySQLbulkLoader("alteration_driver_annotation").insertRecord( + ClickHouseBulkLoader.getClickHouseBulkLoader("alteration_driver_annotation").insertRecord( Long.toString(structuralVariant.getInternalId()), Integer.toString(structuralVariant.getGeneticProfileId()), Integer.toString(structuralVariant.getSampleIdInternal()), @@ -153,20 +153,27 @@ public static void deleteStructuralVariants(int geneticProfileId, Set s ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGene.class); - pstmt = con.prepareStatement("DELETE structural_variant, alteration_driver_annotation" + - " FROM structural_variant" + - " LEFT JOIN alteration_driver_annotation" + - " ON alteration_driver_annotation.GENETIC_PROFILE_ID = structural_variant.GENETIC_PROFILE_ID" + - " AND alteration_driver_annotation.SAMPLE_ID = structural_variant.SAMPLE_ID" + - " WHERE structural_variant.GENETIC_PROFILE_ID=? AND structural_variant.SAMPLE_ID IN (" - + String.join(",", Collections.nCopies(sampleIds.size(), "?")) - + ")"); + if (sampleIds.isEmpty()) { + return; + } + String placeholders = String.join(",", Collections.nCopies(sampleIds.size(), "?")); + pstmt = con.prepareStatement("DELETE FROM alteration_driver_annotation " + + "WHERE genetic_profile_id=? AND sample_id IN (" + placeholders + ")"); int parameterIndex = 1; pstmt.setInt(parameterIndex++, geneticProfileId); for (Integer sampleId : sampleIds) { pstmt.setInt(parameterIndex++, sampleId); } pstmt.executeUpdate(); + pstmt.close(); + pstmt = con.prepareStatement("DELETE FROM structural_variant " + + "WHERE genetic_profile_id=? AND sample_id IN (" + placeholders + ")"); + parameterIndex = 1; + pstmt.setInt(parameterIndex++, geneticProfileId); + for (Integer sampleId : sampleIds) { + pstmt.setInt(parameterIndex++, sampleId); + } + pstmt.executeUpdate(); } catch (SQLException e) { throw new DaoException(e); } finally { @@ -180,7 +187,7 @@ public static long getLargestInternalId() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoMutation.class); - pstmt = con.prepareStatement("SELECT MAX(`INTERNAL_ID`) FROM `structural_variant`"); + pstmt = con.prepareStatement("SELECT max(`internal_id`) FROM `structural_variant`"); rs = pstmt.executeQuery(); return rs.next() ? rs.getLong(1) : 0; } catch (SQLException e) { @@ -205,9 +212,9 @@ public static List getAllStructuralVariants() throws DaoExcep pstmt = con.prepareStatement( "SELECT * FROM structural_variant" + " LEFT JOIN alteration_driver_annotation ON" + - " structural_variant.GENETIC_PROFILE_ID = alteration_driver_annotation.GENETIC_PROFILE_ID" + - " and structural_variant.SAMPLE_ID = alteration_driver_annotation.SAMPLE_ID" + - " and structural_variant.INTERNAL_ID = alteration_driver_annotation.ALTERATION_EVENT_ID"); + " structural_variant.genetic_profile_id = alteration_driver_annotation.genetic_profile_id" + + " and structural_variant.sample_id = alteration_driver_annotation.sample_id" + + " and structural_variant.internal_id = alteration_driver_annotation.alteration_event_id"); rs = pstmt.executeQuery(); while (rs.next()) { @@ -230,49 +237,49 @@ public static List getAllStructuralVariants() throws DaoExcep */ private static StructuralVariant extractStructuralVariant(ResultSet rs) throws SQLException { StructuralVariant structuralVariant = new StructuralVariant(); - structuralVariant.setGeneticProfileId(rs.getInt("GENETIC_PROFILE_ID")); - structuralVariant.setSampleIdInternal(rs.getInt("SAMPLE_ID")); - structuralVariant.setSite1EntrezGeneId(rs.getLong("SITE1_ENTREZ_GENE_ID")); - structuralVariant.setSite1EnsemblTranscriptId(rs.getString("SITE1_ENSEMBL_TRANSCRIPT_ID")); - structuralVariant.setSite1Chromosome(rs.getString("SITE1_CHROMOSOME")); - structuralVariant.setSite1Position(rs.getInt("SITE1_POSITION")); - structuralVariant.setSite1Contig(rs.getString("SITE1_CONTIG")); - structuralVariant.setSite1Region(rs.getString("SITE1_REGION")); - structuralVariant.setSite1RegionNumber(rs.getInt("SITE1_REGION_NUMBER")); - structuralVariant.setSite1Description(rs.getString("SITE1_DESCRIPTION")); - structuralVariant.setSite2EntrezGeneId(rs.getLong("SITE2_ENTREZ_GENE_ID")); - structuralVariant.setSite2EnsemblTranscriptId(rs.getString("SITE2_ENSEMBL_TRANSCRIPT_ID")); - structuralVariant.setSite2Chromosome(rs.getString("SITE2_CHROMOSOME")); - structuralVariant.setSite2Position(rs.getInt("SITE2_POSITION")); - structuralVariant.setSite2Contig(rs.getString("SITE2_CONTIG")); - structuralVariant.setSite2Region(rs.getString("SITE2_REGION")); - structuralVariant.setSite2RegionNumber(rs.getInt("SITE2_REGION_NUMBER")); - structuralVariant.setSite2Description(rs.getString("SITE2_DESCRIPTION")); - structuralVariant.setSite2EffectOnFrame(rs.getString("SITE2_EFFECT_ON_FRAME")); - structuralVariant.setNcbiBuild(rs.getString("NCBI_BUILD")); - structuralVariant.setDnaSupport(rs.getString("DNA_SUPPORT")); - structuralVariant.setRnaSupport(rs.getString("RNA_SUPPORT")); - structuralVariant.setNormalReadCount(rs.getInt("NORMAL_READ_COUNT")); - structuralVariant.setTumorReadCount(rs.getInt("TUMOR_READ_COUNT")); - structuralVariant.setNormalVariantCount(rs.getInt("NORMAL_VARIANT_COUNT")); - structuralVariant.setTumorVariantCount(rs.getInt("TUMOR_VARIANT_COUNT")); - structuralVariant.setNormalPairedEndReadCount(rs.getInt("NORMAL_PAIRED_END_READ_COUNT")); - structuralVariant.setTumorPairedEndReadCount(rs.getInt("TUMOR_PAIRED_END_READ_COUNT")); - structuralVariant.setNormalSplitReadCount(rs.getInt("NORMAL_SPLIT_READ_COUNT")); - structuralVariant.setTumorSplitReadCount(rs.getInt("TUMOR_SPLIT_READ_COUNT")); - structuralVariant.setAnnotation(rs.getString("ANNOTATION")); - structuralVariant.setBreakpointType(rs.getString("BREAKPOINT_TYPE")); - structuralVariant.setConnectionType(rs.getString("CONNECTION_TYPE")); - structuralVariant.setEventInfo(rs.getString("EVENT_INFO")); - structuralVariant.setVariantClass(rs.getString("CLASS")); - structuralVariant.setLength(rs.getInt("LENGTH")); - structuralVariant.setComments(rs.getString("COMMENTS")); - structuralVariant.setDriverFilter(rs.getString("DRIVER_FILTER")); - structuralVariant.setDriverFilterAnn(rs.getString("DRIVER_FILTER_ANNOTATION")); - structuralVariant.setDriverTiersFilter(rs.getString("DRIVER_TIERS_FILTER")); - structuralVariant.setDriverTiersFilterAnn(rs.getString("DRIVER_TIERS_FILTER_ANNOTATION")); - structuralVariant.setSvStatus(rs.getString("SV_STATUS")); - structuralVariant.setAnnotationJson(rs.getString("ANNOTATION_JSON")); + structuralVariant.setGeneticProfileId(rs.getInt("genetic_profile_id")); + structuralVariant.setSampleIdInternal(rs.getInt("sample_id")); + structuralVariant.setSite1EntrezGeneId(rs.getLong("site1_entrez_gene_id")); + structuralVariant.setSite1EnsemblTranscriptId(rs.getString("site1_ensembl_transcript_id")); + structuralVariant.setSite1Chromosome(rs.getString("site1_chromosome")); + structuralVariant.setSite1Position(rs.getInt("site1_position")); + structuralVariant.setSite1Contig(rs.getString("site1_contig")); + structuralVariant.setSite1Region(rs.getString("site1_region")); + structuralVariant.setSite1RegionNumber(rs.getInt("site1_region_number")); + structuralVariant.setSite1Description(rs.getString("site1_description")); + structuralVariant.setSite2EntrezGeneId(rs.getLong("site2_entrez_gene_id")); + structuralVariant.setSite2EnsemblTranscriptId(rs.getString("site2_ensembl_transcript_id")); + structuralVariant.setSite2Chromosome(rs.getString("site2_chromosome")); + structuralVariant.setSite2Position(rs.getInt("site2_position")); + structuralVariant.setSite2Contig(rs.getString("site2_contig")); + structuralVariant.setSite2Region(rs.getString("site2_region")); + structuralVariant.setSite2RegionNumber(rs.getInt("site2_region_number")); + structuralVariant.setSite2Description(rs.getString("site2_description")); + structuralVariant.setSite2EffectOnFrame(rs.getString("site2_effect_on_frame")); + structuralVariant.setNcbiBuild(rs.getString("ncbi_build")); + structuralVariant.setDnaSupport(rs.getString("dna_support")); + structuralVariant.setRnaSupport(rs.getString("rna_support")); + structuralVariant.setNormalReadCount(rs.getInt("normal_read_count")); + structuralVariant.setTumorReadCount(rs.getInt("tumor_read_count")); + structuralVariant.setNormalVariantCount(rs.getInt("normal_variant_count")); + structuralVariant.setTumorVariantCount(rs.getInt("tumor_variant_count")); + structuralVariant.setNormalPairedEndReadCount(rs.getInt("normal_paired_end_read_count")); + structuralVariant.setTumorPairedEndReadCount(rs.getInt("tumor_paired_end_read_count")); + structuralVariant.setNormalSplitReadCount(rs.getInt("normal_split_read_count")); + structuralVariant.setTumorSplitReadCount(rs.getInt("tumor_split_read_count")); + structuralVariant.setAnnotation(rs.getString("annotation")); + structuralVariant.setBreakpointType(rs.getString("breakpoint_type")); + structuralVariant.setConnectionType(rs.getString("connection_type")); + structuralVariant.setEventInfo(rs.getString("event_info")); + structuralVariant.setVariantClass(rs.getString("class")); + structuralVariant.setLength(rs.getInt("length")); + structuralVariant.setComments(rs.getString("comments")); + structuralVariant.setDriverFilter(rs.getString("driver_filter")); + structuralVariant.setDriverFilterAnn(rs.getString("driver_filter_annotation")); + structuralVariant.setDriverTiersFilter(rs.getString("driver_tiers_filter")); + structuralVariant.setDriverTiersFilterAnn(rs.getString("driver_tiers_filter_annotation")); + structuralVariant.setSvStatus(rs.getString("sv_status")); + structuralVariant.setAnnotationJson(rs.getString("annotation_json")); return structuralVariant; } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoTypeOfCancer.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoTypeOfCancer.java index aad29636..c9dd42af 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoTypeOfCancer.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoTypeOfCancer.java @@ -52,7 +52,7 @@ public static int addTypeOfCancer(TypeOfCancer typeOfCancer) throws DaoException ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoTypeOfCancer.class); - pstmt = con.prepareStatement("INSERT INTO type_of_cancer ( `TYPE_OF_CANCER_ID`, `NAME`, `DEDICATED_COLOR`, `SHORT_NAME`, `PARENT` ) VALUES (?,?,?,?,?)"); + pstmt = con.prepareStatement("INSERT INTO type_of_cancer ( `type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent` ) VALUES (?,?,?,?,?)"); pstmt.setString(1, typeOfCancer.getTypeOfCancerId()); pstmt.setString(2, typeOfCancer.getName()); pstmt.setString(3, typeOfCancer.getDedicatedColor()); @@ -67,13 +67,16 @@ public static int addTypeOfCancer(TypeOfCancer typeOfCancer) throws DaoException } public static TypeOfCancer getTypeOfCancerById(String typeOfCancerId) throws DaoException { + if (typeOfCancerId == null) { + return null; + } Connection con = null; PreparedStatement pstmt = null; ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoTypeOfCancer.class); - pstmt = con.prepareStatement("SELECT * FROM type_of_cancer WHERE TYPE_OF_CANCER_ID=?"); - pstmt.setString(1, typeOfCancerId); + pstmt = con.prepareStatement("SELECT * FROM type_of_cancer WHERE LOWER(type_of_cancer_id)=?"); + pstmt.setString(1, typeOfCancerId.toLowerCase()); rs = pstmt.executeQuery(); if (rs.next()) { return extractTypeOfCancer(rs); @@ -113,7 +116,7 @@ public static int getCount() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoTypeOfCancer.class); - pstmt = con.prepareStatement("SELECT COUNT(*) FROM type_of_cancer"); + pstmt = con.prepareStatement("SELECT count(*) FROM type_of_cancer"); rs = pstmt.executeQuery(); if (rs.next()) { return rs.getInt(1); @@ -139,10 +142,8 @@ public static void deleteAllRecords() throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoTypeOfCancer.class); - JdbcUtil.disableForeignKeyCheck(con); pstmt = con.prepareStatement("TRUNCATE TABLE type_of_cancer"); pstmt.executeUpdate(); - JdbcUtil.enableForeignKeyCheck(con); } catch (SQLException e) { throw new DaoException(e); } finally { @@ -168,7 +169,7 @@ public static void deleteTypeOfCancer(String typeOfCancerId) throws DaoException ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoTypeOfCancer.class); - pstmt = con.prepareStatement("DELETE from " + "type_of_cancer WHERE TYPE_OF_CANCER_ID=?"); + pstmt = con.prepareStatement("DELETE from " + "type_of_cancer WHERE type_of_cancer_id=?"); pstmt.setString(1, typeOfCancerId); pstmt.executeUpdate(); } catch (SQLException e) { @@ -181,11 +182,11 @@ public static void deleteTypeOfCancer(String typeOfCancerId) throws DaoException private static TypeOfCancer extractTypeOfCancer(ResultSet rs) throws SQLException { TypeOfCancer typeOfCancer = new TypeOfCancer(); - typeOfCancer.setTypeOfCancerId(rs.getString("TYPE_OF_CANCER_ID")); - typeOfCancer.setName(rs.getString("NAME")); - typeOfCancer.setDedicatedColor(rs.getString("DEDICATED_COLOR")); - typeOfCancer.setShortName(rs.getString("SHORT_NAME")); - typeOfCancer.setParentTypeOfCancerId(rs.getString("PARENT")); + typeOfCancer.setTypeOfCancerId(rs.getString("type_of_cancer_id")); + typeOfCancer.setName(rs.getString("name")); + typeOfCancer.setDedicatedColor(rs.getString("dedicated_color")); + typeOfCancer.setShortName(rs.getString("short_name")); + typeOfCancer.setParentTypeOfCancerId(rs.getString("parent")); return typeOfCancer; } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoUser.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoUser.java index 1ff3cb1d..56581219 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoUser.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoUser.java @@ -52,7 +52,7 @@ public static int addUser(User user) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoUser.class); - pstmt = con.prepareStatement("INSERT INTO users ( `EMAIL`, `NAME`, `ENABLED` ) VALUES (?,?,?)"); + pstmt = con.prepareStatement("INSERT INTO users ( `email`, `name`, `enabled` ) VALUES (?,?,?)"); pstmt.setString(1, user.getEmail()); pstmt.setString(2, user.getName()); pstmt.setBoolean(3, user.isEnabled()); @@ -78,7 +78,7 @@ public static User getUserByEmail(String email) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoUser.class); - pstmt = con.prepareStatement("SELECT * FROM users WHERE EMAIL=?"); + pstmt = con.prepareStatement("SELECT * FROM users WHERE email=?"); pstmt.setString(1, email); rs = pstmt.executeQuery(); if (rs.next()) { @@ -134,7 +134,7 @@ public static void deleteUser(String email) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoUser.class); - pstmt = con.prepareStatement("DELETE FROM users WHERE EMAIL=?"); + pstmt = con.prepareStatement("DELETE FROM users WHERE email=?"); pstmt.setString(1, email); pstmt.executeUpdate(); } catch (SQLException e) { @@ -146,8 +146,8 @@ public static void deleteUser(String email) throws DaoException { private static User extractUser(ResultSet rs) throws SQLException { - return new User(rs.getString("EMAIL"), - rs.getString("NAME"), - rs.getBoolean("ENABLED")); + return new User(rs.getString("email"), + rs.getString("name"), + rs.getBoolean("enabled")); } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoUserAuthorities.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoUserAuthorities.java index 0b13c513..5f163388 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoUserAuthorities.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoUserAuthorities.java @@ -53,7 +53,7 @@ public static int addUserAuthorities(UserAuthorities userAuthorities) throws Dao con = JdbcUtil.getDbConnection(DaoUserAuthorities.class); String email = userAuthorities.getEmail(); for (String authority : userAuthorities.getAuthorities()) { - pstmt = con.prepareStatement("INSERT INTO authorities (`EMAIL`, `AUTHORITY`) VALUES (?,?)"); + pstmt = con.prepareStatement("INSERT INTO authorities (`email`, `authority`) VALUES (?,?)"); pstmt.setString(1, email); pstmt.setString(2, authority); toReturn += pstmt.executeUpdate(); @@ -74,12 +74,12 @@ public static UserAuthorities getUserAuthorities(User user) throws DaoException ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoUserAuthorities.class); - pstmt = con.prepareStatement("SELECT * FROM authorities where EMAIL=?"); + pstmt = con.prepareStatement("SELECT * FROM authorities where email=?"); pstmt.setString(1, user.getEmail()); rs = pstmt.executeQuery(); ArrayList authorities = new ArrayList(); while (rs.next()) { - authorities.add(rs.getString("AUTHORITY")); + authorities.add(rs.getString("authority")); } return new UserAuthorities(user.getEmail(), authorities); } catch (SQLException e) { @@ -95,7 +95,7 @@ public static void removeUserAuthorities(User user) throws DaoException { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoUserAuthorities.class); - pstmt = con.prepareStatement("DELETE FROM authorities where EMAIL=?"); + pstmt = con.prepareStatement("DELETE FROM authorities where email=?"); pstmt.setString(1, user.getEmail()); pstmt.executeUpdate(); } catch (SQLException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java b/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java index 7dbd668e..642c6d4f 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/JdbcDataSource.java @@ -44,7 +44,7 @@ public JdbcDataSource() { String username = dbProperties.getSpringDbUser(); String password = dbProperties.getSpringDbPassword(); String connectionURL = dbProperties.getSpringConnectionURL(); - String mysqlDriverClassName = dbProperties.getSpringDbDriverClassName(); + String driverClassName = dbProperties.getSpringDbDriverClassName(); // extract optional property values String enablePooling = dbProperties.getDbEnablePooling(); if (stringIsNullOrBlank(enablePooling)) { @@ -55,18 +55,15 @@ public JdbcDataSource() { requiredPropertyInfoList.add(new RequiredPropertyInfo("spring.datasource.username", username, "username")); requiredPropertyInfoList.add(new RequiredPropertyInfo("spring.datasource.password", password, "password")); requiredPropertyInfoList.add(new RequiredPropertyInfo("spring.datasource.url", connectionURL, "connectionURL")); - requiredPropertyInfoList.add(new RequiredPropertyInfo("spring.datasource.driver-class-name", mysqlDriverClassName, "driver class name")); + requiredPropertyInfoList.add(new RequiredPropertyInfo("spring.datasource.driver-class-name", driverClassName, "driver class name")); validateRequiredProperties(requiredPropertyInfoList); // Set up poolable data source this.setUsername(username); this.setPassword(password); this.setUrl(connectionURL); - this.setDriverClassName(mysqlDriverClassName); + this.setDriverClassName(driverClassName); // Disable this to avoid caching statements this.setPoolPreparedStatements(Boolean.valueOf(enablePooling)); - // this property setting is needed to enable MySQLbulkLoader to load data from a local file - // this is set here in case this class is initialized via JdbcUtil (not expected), but is also set in applicationContext resource files. - this.addConnectionProperty("allowLoadLocalInfile", "true"); // these values are from the production cbioportal application context for a jndi data source this.setMaxTotal(500); this.setMaxIdle(30); diff --git a/src/main/java/org/mskcc/cbio/portal/dao/JdbcUtil.java b/src/main/java/org/mskcc/cbio/portal/dao/JdbcUtil.java index f8fd802e..40cda419 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/JdbcUtil.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/JdbcUtil.java @@ -145,7 +145,7 @@ private static void closeConnection(String requester, Connection con) { } } } catch (Exception e) { - logMessage("Problem Closed a MySQL connection from " + requester + ": " + activeConnectionCount.toString()); + logMessage("Problem closing a database connection from " + requester + ": " + activeConnectionCount.toString()); e.printStackTrace(); } } @@ -236,33 +236,4 @@ static Long readLongFromResultSet(ResultSet rs, String column) throws SQLExcepti long l = rs.getInt(column); return rs.wasNull() ? null : l; } - - static Double readDoubleFromResultSet(ResultSet rs, String column) throws SQLException { - double d = rs.getDouble(column); - return rs.wasNull() ? null : d; - } - - /** - * Tells the database to ignore foreign key constraints, effective only for current session. - * Useful when you want to truncate a table that has foreign key constraints. Note that this - * may create orphan records in child tables. - * @param con Database connection - * @throws SQLException - */ - public static void disableForeignKeyCheck(Connection con) throws SQLException { - Statement stmt = con.createStatement(); - stmt.execute("SET FOREIGN_KEY_CHECKS=0"); - stmt.close(); - } - - /** - * Reverses the effect of disableForeignKeyCheck method. - * @param con Database Connection - * @throws SQLException - */ - public static void enableForeignKeyCheck(Connection con) throws SQLException { - Statement stmt = con.createStatement(); - stmt.execute("SET FOREIGN_KEY_CHECKS=1"); - stmt.close(); - } } diff --git a/src/main/java/org/mskcc/cbio/portal/dao/MySQLbulkLoader.java b/src/main/java/org/mskcc/cbio/portal/dao/MySQLbulkLoader.java deleted file mode 100644 index cb12ac28..00000000 --- a/src/main/java/org/mskcc/cbio/portal/dao/MySQLbulkLoader.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Copyright (c) 2015 Memorial Sloan-Kettering Cancer Center. - * - * This library is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS - * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder - * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no - * obligations to provide maintenance, support, updates, enhancements or - * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be - * liable to any party for direct, indirect, special, incidental or - * consequential damages, including lost profits, arising out of the use of this - * software and its documentation, even if Memorial Sloan-Kettering Cancer - * Center has been advised of the possibility of such damage. - */ - -/* - * This file is part of cBioPortal. - * - * cBioPortal is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . -*/ - -package org.mskcc.cbio.portal.dao; - -import java.io.*; -import java.sql.*; -import java.util.*; -import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; -import org.mskcc.cbio.portal.util.FileUtil; -import org.mskcc.cbio.portal.util.ProgressMonitor; - -/** - * To speed up CGDS data loading, bulk load from files using MySQL "LOAD DATA INFILE" functionality. - * Intercept each record write in the normal load, buffer it in a temp file, and load the temp file when done. - * NOT thread-safe. - * @author arthur goldberg - * In the future, would be cooler to implement this by overloading the JDBC Connection.prepareStatement - * and PreparedStatement.setX() calls. - */ -public class MySQLbulkLoader { - private static boolean bulkLoad = false; - private static boolean relaxedMode = false; - private String[] fieldNames = null; - - private static final Map mySQLbulkLoaders = new LinkedHashMap(); - /** - * Get a MySQLbulkLoader - * @param dbName database name - * @return - */ - public static MySQLbulkLoader getMySQLbulkLoader(String dbName) { - MySQLbulkLoader mySQLbulkLoader = mySQLbulkLoaders.get(dbName); - if (mySQLbulkLoader==null) { - mySQLbulkLoader = new MySQLbulkLoader(dbName); - mySQLbulkLoaders.put(dbName, mySQLbulkLoader); - } - return mySQLbulkLoader; - } - - /** - * Flushes all pending data from the bulk writer. Temporarily disables referential - * integrity while it does so, largely because MySQL uses a weird model and expects - * referential integrity at each record, not just at the end of the transaction. - * @return the number of rows added - * @throws DaoException - */ - public static int flushAll() throws DaoException { - int checks = 0; - PreparedStatement stmt = null; - boolean executedSetFKChecks = false; - Connection con = null; - try { - con = JdbcUtil.getDbConnection(MySQLbulkLoader.class); - stmt = con.prepareStatement("SELECT @@foreign_key_checks;"); - ResultSet result = stmt.executeQuery(); - result.next(); - checks = result.getInt(1); - stmt = con.prepareStatement("SET foreign_key_checks = ?;"); - stmt.setLong(1, 0); - stmt.execute(); - executedSetFKChecks = true; - - int n = 0; - for (MySQLbulkLoader mySQLbulkLoader : mySQLbulkLoaders.values()) { - n += mySQLbulkLoader.loadDataFromTempFileIntoDBMS(); - } - - return n; - } catch (IOException e) { - System.err.println("Could not open temp file"); - e.printStackTrace(); - return -1; - } catch (SQLException e) { - throw new DaoException(e); - } - finally { - mySQLbulkLoaders.clear(); - if (executedSetFKChecks && stmt != null) { - try { - stmt.setLong(1, checks); - stmt.execute(); - } - catch (SQLException e) { - throw new DaoException(e); - } - } - JdbcUtil.closeAll(MySQLbulkLoader.class, con, stmt, null); - } - } - - private String tempFileName = null; - private File tempFileHandle = null; - private BufferedWriter tempFileWriter = null; - private String tableName; - private final String tempTableSuffix = ".tempTable"; - private int rows; - // TODO: make configurable - private static final long numDebuggingRowsToPrint = 0; - - private MySQLbulkLoader( String tableName ){ - try { - openTempFile( tableName ); - this.tableName = tableName; - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * Open temp file for table 'tableName'. - * note that auto_increment fields must be handled specially. - * @param tableName - * @throws FileNotFoundException - * @throws IOException - * @throws IllegalArgumentException - */ - private void openTempFile(String tableName) throws IOException { - - tempFileHandle = File.createTempFile( tableName, tempTableSuffix, FileUtils.getTempDirectory() ); - - tempFileName = tempFileHandle.getAbsolutePath(); - - if (!tempFileHandle.exists()) { - throw new FileNotFoundException("File does not exist: " + tempFileHandle); - } - if (!tempFileHandle.isFile()) { - throw new IllegalArgumentException("Should not be a directory: " + tempFileHandle); - } - if (!tempFileHandle.canWrite()) { - throw new IllegalArgumentException("File cannot be written: " + tempFileHandle); - } - - // to improve performance use buffering; FileWriter always assumes default encoding is OK! - this.tempFileWriter = new BufferedWriter(new FileWriter(tempFileHandle, false)); - } - - /** - * write a record's fields, in order, to the table's temp file. if no fields are provided, writes no record. - * fields are TAB separated. - * @param fieldValues - */ - public void insertRecord( String... fieldValues) { - if( fieldValues.length == 0 ){ - return; - } - try { - tempFileWriter.write( escapeValue(fieldValues[0]) ); - for( int i=1; i records inserted into `" + tableName + "` table: " + updateCount); - int nLines = FileUtil.getNumLines(tempFileHandle); - if (nLines!=updateCount && !relaxedMode) { - String otherDetails = ""; - if (stmt.getWarnings() != null) { - otherDetails = "More error/warning details: " + stmt.getWarnings().getMessage(); - } - throw new DaoException("DB Error: only "+updateCount+" of the "+nLines+" records were inserted in `" + tableName + "`. " + otherDetails + - " See tmp file for more details: " + tempFileName); - - } else { - tempFileHandle.delete(); - } - - // reopen empty temp file -- not necessary, this loader will be removed. - //this.tempFileWriter = new BufferedWriter(new FileWriter( this.tempFileHandle, false)); - - return updateCount; - - } catch (SQLException e) { - throw new DaoException(e); - } finally { - JdbcUtil.closeAll(MySQLbulkLoader.class, con, pstmt, rs); - } - } - - public String getTempFileName() { - return tempFileName; - } - - public String getTableName() { - return tableName; - } - - public static boolean isBulkLoad() { - return bulkLoad; - } - - public static void bulkLoadOn() { - MySQLbulkLoader.bulkLoad = true; - } - - public static void bulkLoadOff() { - MySQLbulkLoader.bulkLoad = false; - } - - public static void relaxedModeOn() { - MySQLbulkLoader.relaxedMode = true; - } - - public static void relaxedModeOff() { - MySQLbulkLoader.relaxedMode = false; - } - - public void setFieldNames(String[] fieldNames) { - this.fieldNames = fieldNames; - } - -} diff --git a/src/main/java/org/mskcc/cbio/portal/model/ResourceBaseData.java b/src/main/java/org/mskcc/cbio/portal/model/ResourceBaseData.java index 27f4ebc4..f60aa8bf 100644 --- a/src/main/java/org/mskcc/cbio/portal/model/ResourceBaseData.java +++ b/src/main/java/org/mskcc/cbio/portal/model/ResourceBaseData.java @@ -8,12 +8,14 @@ public class ResourceBaseData { private String stableId; private String resourceId; private String url; + private String metadata; + private String groupPath; /** * Constructor */ public ResourceBaseData() { - this(-1, "", "", ""); + this(-1, "", "", "", null, null); } /** @@ -24,16 +26,21 @@ public ResourceBaseData() { * @param resourceId resource id * @param url url of the resource */ - public ResourceBaseData(int cancerStudyId, String stableId, String resourceId, String url) { + this(cancerStudyId, stableId, resourceId, url, null, null); + } + + public ResourceBaseData(int cancerStudyId, String stableId, String resourceId, String url, String metadata, String groupPath) { this.setCancerStudyId(cancerStudyId); this.setStableId(stableId); this.setResourceId(resourceId); this.setUrl(url); + this.setMetadata(metadata); + this.setGroupPath(groupPath); } public ResourceBaseData(ResourceBaseData other) { - this(other.getCancerStudyId(), other.getStableId(), other.getResourceId(), other.getUrl()); + this(other.getCancerStudyId(), other.getStableId(), other.getResourceId(), other.getUrl(), other.getMetadata(), other.getGroupPath()); } public int getCancerStudyId() { @@ -67,4 +74,20 @@ public String getUrl() { public void setUrl(String url) { this.url = url; } + + public String getMetadata() { + return metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } + + public String getGroupPath() { + return groupPath; + } + + public void setGroupPath(String groupPath) { + this.groupPath = groupPath; + } } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/GeneticAlterationIncrementalImporter.java b/src/main/java/org/mskcc/cbio/portal/scripts/GeneticAlterationIncrementalImporter.java index 8302990b..8c5345aa 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/GeneticAlterationIncrementalImporter.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/GeneticAlterationIncrementalImporter.java @@ -46,7 +46,7 @@ public void initialize() { throw new IllegalStateException("Number of samples (" + sampleToValue.size() + ") for genetic entity with id " + geneticEntityId + " does not match with the number in the preexisting sample list (" - + initialOrderSampleListSize + ")."); + + initialOrderSampleListSize + "). Consider reuploading the entire study instead of doing an incremental update."); } }); // add all new sample ids at the end @@ -54,8 +54,6 @@ public void initialize() { Set savedSampleSet = new HashSet<>(savedOrderedSampleList); List newSampleIds = this.fileOrderedSampleList.stream().filter(sampleId -> !savedSampleSet.contains(sampleId)).toList(); this.orderedSampleList.addAll(newSampleIds); - DaoGeneticProfileSamples.deleteAllSamplesInGeneticProfile(this.geneticProfileId); - daoGeneticAlteration.deleteAllRecordsInGeneticProfile(this.geneticProfileId); } catch (DaoException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportClinicalData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportClinicalData.java index efce8dcd..48a97676 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportClinicalData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportClinicalData.java @@ -39,16 +39,16 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.apache.commons.collections4.map.MultiKeyMap; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalAttributeMeta; import org.mskcc.cbio.portal.dao.DaoClinicalData; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ClinicalAttribute; +import org.mskcc.cbio.portal.model.ClinicalData; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.util.FileUtil; @@ -161,10 +161,10 @@ public void importData() throws Exception // code has to be added to check whether // a clinical attribute update should be // perform instead of an insert - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); if(relaxed) { - MySQLbulkLoader.relaxedModeOn(); + ClickHouseBulkLoader.relaxedModeOn(); } FileReader reader = new FileReader(clinicalDataFile); @@ -188,16 +188,15 @@ public void importData() throws Exception " in file. Please check your file format and try again."); } importData(buff, columnAttrs); - + + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.relaxedModeOff(); + } if (getAttributesType() == ImportClinicalData.AttributeTypes.SAMPLE_ATTRIBUTES || getAttributesType() == ImportClinicalData.AttributeTypes.MIXED_ATTRIBUTES) { DaoPatient.createSampleCountClinicalData(cancerStudy.getInternalId()); } - - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); - MySQLbulkLoader.relaxedModeOff(); - } } private List grabAttrs(BufferedReader buff) throws DaoException, IOException { @@ -295,15 +294,16 @@ private String[] splitFields(String line) throws IOException { private void importData(BufferedReader buff, List columnAttrs) throws Exception { String line; - MultiKeyMap attributeMap = new MultiKeyMap(); + Map alreadyParsedAttributes = new HashMap<>(); while ((line = buff.readLine()) != null) { if (skipLine(line.trim())) { continue; } String[] fieldValues = getFieldValues(line, columnAttrs); - addDatum(fieldValues, columnAttrs, attributeMap); + alreadyParsedAttributes.putAll(parseRowAttributes(fieldValues, columnAttrs, alreadyParsedAttributes)); } + addClinicalDataIfNoDuplicates(alreadyParsedAttributes); } private boolean skipLine(String line) @@ -338,7 +338,9 @@ private String[] getFieldValues(String line, List columnAttrs return fieldValues; } - private boolean addDatum(String[] fields, List columnAttrs, MultiKeyMap attributeMap) throws Exception + private record FullAttributeKey(String table, int internalId, String attrId) {} + + private Map parseRowAttributes(String[] fields, List columnAttrs, Map alreadyParsedAttributes) throws Exception { int sampleIdIndex = findSampleIdColumn(columnAttrs); String stableSampleId = (sampleIdIndex >= 0) ? fields[sampleIdIndex] : ""; @@ -360,13 +362,13 @@ private boolean addDatum(String[] fields, List columnAttrs, M //and an ERROR in other studies. I.e. a sample should occur only once in clinical file! if (stableSampleId.startsWith("TCGA-")) { ProgressMonitor.logWarning("Sample " + stableSampleId + " found to be duplicated in your file. Only data of the first sample will be processed."); - return false; + return Collections.emptyMap(); } if (this.isSupplementalData()) { internalSampleId = sample.getInternalId(); } else { //give error or warning if sample is already in DB and this is NOT expected (i.e. not supplemental data): - throw new RuntimeException("Error: Sample " + stableSampleId + " found to be duplicated in your file."); + throw new RuntimeException("Error: Sample " + stableSampleId + " is already in the database."); } } } @@ -401,6 +403,7 @@ private boolean addDatum(String[] fields, List columnAttrs, M numSamplesProcessed++; } + Map attributeRow = new HashMap<>(); for (int lc = 0; lc < fields.length; lc++) { //if lc is sampleIdIndex or patientIdIndex, skip as well since these are the relational fields: if (lc == sampleIdIndex || lc == patientIdIndex) { @@ -411,44 +414,42 @@ private boolean addDatum(String[] fields, List columnAttrs, M numEmptyClinicalAttributesSkipped++; continue; } - boolean isPatientAttribute = columnAttrs.get(lc).isPatientAttribute(); + boolean isPatientAttribute = columnAttrs.get(lc).isPatientAttribute(); if (isPatientAttribute && internalPatientId != -1) { // The attributeMap keeps track what patient/attribute to value pairs are being added to the DB. If there are duplicates, // (which can happen in a MIXED_ATTRIBUTES type clinical file), we need to make sure that the value for the same - // attributes are consistent. This prevents duplicate entries in the temp file that the MySqlBulkLoader uses. - if(!attributeMap.containsKey(internalPatientId, columnAttrs.get(lc).getAttrId())) { - addDatum(internalPatientId, columnAttrs.get(lc).getAttrId(), fields[lc], - ClinicalAttribute.PATIENT_ATTRIBUTE); - attributeMap.put(internalPatientId, columnAttrs.get(lc).getAttrId(), fields[lc]); + // attributes are consistent. This prevents duplicate entries in the temp file that the ClickHouseBulkLoader uses. + FullAttributeKey patientAttributeKey = new FullAttributeKey(ClinicalAttribute.PATIENT_ATTRIBUTE, internalPatientId, columnAttrs.get(lc).getAttrId()); + if(!alreadyParsedAttributes.containsKey(patientAttributeKey)) { + attributeRow.put(patientAttributeKey, fields[lc]); } else if (!relaxed) { - throw new RuntimeException("Error: Duplicated patient in file"); + throw new RuntimeException("Error: Duplicated patient " + stablePatientId + " and " + columnAttrs.get(lc).getAttrId() + " attribute pair in the clinical file."); } - // if the "relaxed" flag was given, and the new record e.g. tries to override a previously set attribute for - // an existing patient (e.g. set AGE from 2 to 10...in same study, or GENDER from M to F), then the system will keep - // the previous value and give a warning. NB: this is a kind of "random" harmonization strategy and is - // NOT recommended! TODO - change this to an Exception instead of just a warning. - else if (!attributeMap.get(internalPatientId, columnAttrs.get(lc).getAttrId()).equals(fields[lc])) { - ProgressMonitor.logWarning("Error: Duplicated patient " + stablePatientId + " with different values for patient attribute " + columnAttrs.get(lc).getAttrId() + - "\n\tValues: " + attributeMap.get(internalPatientId, columnAttrs.get(lc).getAttrId()) + " " + fields[lc]); + // if the "relaxed" flag was given, and the new record e.g. tries to override a previously set attribute for + // an existing patient (e.g. set AGE from 2 to 10...in same study, or GENDER from M to F), then the system will keep + // the previous value and give a warning. NB: this is a kind of "random" harmonization strategy and is + // NOT recommended! TODO - change this to an Exception instead of just a warning. + else if (!alreadyParsedAttributes.get(patientAttributeKey).equals(fields[lc])) { + ProgressMonitor.logWarning("Error: Duplicated patient " + stablePatientId + " with different values for patient attribute " + columnAttrs.get(lc).getAttrId() + + "\n\tValues: " + alreadyParsedAttributes.get(patientAttributeKey) + " " + fields[lc]); } } else if (internalSampleId != -1) { - if(!attributeMap.containsKey(internalSampleId, columnAttrs.get(lc).getAttrId())) { - addDatum(internalSampleId, columnAttrs.get(lc).getAttrId(), fields[lc], - ClinicalAttribute.SAMPLE_ATTRIBUTE); - attributeMap.put(internalSampleId, columnAttrs.get(lc).getAttrId(), fields[lc]); + FullAttributeKey sampleAttributeKey = new FullAttributeKey(ClinicalAttribute.SAMPLE_ATTRIBUTE, internalSampleId, columnAttrs.get(lc).getAttrId()); + if(!alreadyParsedAttributes.containsKey(sampleAttributeKey)) { + attributeRow.put(sampleAttributeKey, fields[lc]); } else if (!relaxed) { - throw new RuntimeException("Error: Duplicated sample in file"); + throw new RuntimeException("Error: Duplicated sample " + stableSampleId + " and " + columnAttrs.get(lc).getAttrId() + " attribute pair in the clinical file."); } - else if (!attributeMap.get(internalSampleId, columnAttrs.get(lc).getAttrId()).equals(fields[lc])) { + else if (!alreadyParsedAttributes.get(sampleAttributeKey).equals(fields[lc])) { ProgressMonitor.logWarning("Error: Duplicated sample " + stableSampleId + " with different values for sample attribute " + columnAttrs.get(lc).getAttrId() + - "\n\tValues: " + attributeMap.get(internalSampleId, columnAttrs.get(lc).getAttrId()) + " " + fields[lc]); + "\n\tValues: " + alreadyParsedAttributes.get(sampleAttributeKey) + " " + fields[lc]); } } } - return true; + return attributeRow; } private boolean isSupplementalData() { @@ -489,6 +490,7 @@ private int addPatientToDatabase(String patientId) throws Exception //in case of PATIENT data import, there are some special checks: if (getAttributesType() == ImportClinicalData.AttributeTypes.PATIENT_ATTRIBUTES) { //if clinical data is already there, then something has gone wrong (e.g. patient is duplicated in file), abort: + //TODO we migth want to optimize this for ClickHouse if (patient != null && DaoClinicalData.getDataByPatientId(cancerStudy.getInternalId(), patientId).size() > 0) { throw new RuntimeException("Something has gone wrong. Patient " + patientId + " already has clinical data loaded."); } @@ -579,6 +581,31 @@ private boolean validSampleId(String sampleId) return (sampleId != null && !sampleId.isEmpty()); } + private void addClinicalDataIfNoDuplicates(Map attributeMap) throws Exception + { + List> internalPatientIdAttributes = attributeMap.keySet().stream().filter(key -> ClinicalAttribute.PATIENT_ATTRIBUTE.equals(key.table)).map(k -> Map.entry(k.internalId, k.attrId)).toList(); + if (!internalPatientIdAttributes.isEmpty()) { + List> existingInternalPatientIdAttributes = DaoClinicalData.getExistingPatientAttributes(internalPatientIdAttributes); + if (!existingInternalPatientIdAttributes.isEmpty()) { + throw new RuntimeException("Error: The following patient internal id and attribute entries already exist in the database: " + + existingInternalPatientIdAttributes); + } + } + List> internalSampleIdAttributes = attributeMap.keySet().stream().filter(key -> ClinicalAttribute.SAMPLE_ATTRIBUTE.equals(key.table)).map(k -> Map.entry(k.internalId, k.attrId)).toList(); + if (!internalSampleIdAttributes.isEmpty()) { + List> existingInternalSampleIdAttributes = DaoClinicalData.getExistingSampleAttributes(internalPatientIdAttributes); + if (!existingInternalSampleIdAttributes.isEmpty()) { + throw new RuntimeException("Error: The following sample internal id and attribute entries already exist in the database: " + + existingInternalSampleIdAttributes); + } + } + for (Map.Entry entry : attributeMap.entrySet()) { + FullAttributeKey key = entry.getKey(); + String value = entry.getValue(); + addDatum(key.internalId, key.attrId, value, key.table); + } + } + private void addDatum(int internalId, String attrId, String attrVal, String attrType) throws Exception { // if bulk loading is ever turned off, we need to check if diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportCnaDiscreteLongData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportCnaDiscreteLongData.java index be9a50df..685de5ae 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportCnaDiscreteLongData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportCnaDiscreteLongData.java @@ -28,6 +28,8 @@ import java.io.*; import java.util.*; import java.util.stream.Collectors; + +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCnaEvent; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; @@ -35,7 +37,6 @@ import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.dao.JdbcUtil; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.CnaEvent; import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; @@ -129,7 +130,7 @@ private void doImportData() throws Exception { if (isDiscretizedCnaProfile) { existingCnaEvents.addAll(DaoCnaEvent.getAllCnaEvents()); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } CnaImportData toImport = new CnaImportData(); @@ -163,7 +164,7 @@ private void doImportData() throws Exception { ProgressMonitor.setCurrentMessage(" --> total number of samples skipped (normal samples): " + getSamplesSkipped()); buf.close(); geneticAlterationGeneImporter.finalize(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } /** diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportCopyNumberSegmentData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportCopyNumberSegmentData.java index 4eeb4165..0fd7487a 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportCopyNumberSegmentData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportCopyNumberSegmentData.java @@ -36,12 +36,12 @@ import java.math.BigDecimal; import java.util.*; import joptsimple.OptionSet; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoCopyNumberSegment; import org.mskcc.cbio.portal.dao.DaoCopyNumberSegmentFile; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CopyNumberSegment; import org.mskcc.cbio.portal.model.CopyNumberSegmentFile; @@ -143,12 +143,12 @@ public void run() { if (!isIncrementalUpdateMode && segmentDataExistsForCancerStudy(cancerStudy)) { throw new IllegalArgumentException("Seg data for cancer study " + cancerStudy.getCancerStudyStableId() + " has already been imported: " + dataFile); } - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); importCopyNumberSegmentFileMetadata(cancerStudy, properties); importCopyNumberSegmentFileData(cancerStudy, dataFile); - MySQLbulkLoader.flushAll(); - MySQLbulkLoader.bulkLoadOff(); DaoCopyNumberSegment.createFractionGenomeAlteredClinicalData(cancerStudy.getInternalId(), processedSampleIds, isIncrementalUpdateMode); + ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.bulkLoadOff(); } catch (RuntimeException e) { throw e; } catch (IOException|DaoException e) { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportExtendedMutationData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportExtendedMutationData.java index aa16aedb..18edd719 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportExtendedMutationData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportExtendedMutationData.java @@ -38,6 +38,7 @@ import org.apache.commons.lang3.StringUtils; import org.mskcc.cbio.maf.MafRecord; import org.mskcc.cbio.maf.MafUtil; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoAlleleSpecificCopyNumber; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; @@ -47,7 +48,6 @@ import org.mskcc.cbio.portal.dao.DaoReferenceGenome; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.AlleleSpecificCopyNumber; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; @@ -132,11 +132,16 @@ public void setSwissprotIsAccession(boolean swissprotIsAccession) { } public void importData() throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); HashSet sequencedCaseSet = new HashSet(); Map existingEvents = new HashMap(); + ProgressMonitor.setCurrentMessage("Starting to load existing mutation events..."); + for(MutationEvent mutationEvent: DaoMutation.getAllMutationEvents()) { + existingEvents.put(mutationEvent, mutationEvent); + } + ProgressMonitor.setCurrentMessage("Loaded " + existingEvents.size() + " existing mutation events."); Set newEvents = new HashSet(); Map mutations = new HashMap(); @@ -328,7 +333,7 @@ public void importData() throws IOException, DaoException { !mutationType.equalsIgnoreCase("IGR")) { ProgressMonitor.logWarning( "Treating mutation with gene symbol 'Unknown' " + - (mafUtil.getEntrezGeneIdIndex() == -1 ? "" : "and Entrez gene ID 0") + " as intergenic ('IGR') " + + (mafUtil.getEntrezGeneIdIndex() == -1 ? "" : "and Entrez gene id 0") + " as intergenic ('IGR') " + "instead of '" + mutationType + "'. Entry filtered/skipped."); } // treat as IGR: @@ -426,10 +431,7 @@ public void importData() throws IOException, DaoException { // Filter out Mutations if( myMutationFilter.acceptMutation( mutation, this.filteredMutations )) { - MutationEvent event = - existingEvents.containsKey(mutation.getEvent()) ? - existingEvents.get(mutation.getEvent()) : - DaoMutation.getMutationEvent(mutation.getEvent()); + MutationEvent event = existingEvents.get(mutation.getEvent()); if (event!=null) { mutation.setEvent(event); } else { @@ -488,8 +490,8 @@ public void importData() throws IOException, DaoException { } } - if( MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); + if( ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); } // run sanity check on `mutation_event` to determine whether duplicate // events were introduced during current import @@ -510,8 +512,8 @@ public void importData() throws IOException, DaoException { ProgressMonitor.setCurrentMessage("Calculating mutation counts by keyword..."); DaoMutation.calculateMutationCountByKeyword(geneticProfileId); - if( MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); + if( ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); } if (entriesSkipped > 0) { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGeneData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGeneData.java index 1881af49..1a3fafd1 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGeneData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGeneData.java @@ -38,11 +38,11 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; import org.mskcc.cbio.portal.dao.DaoReferenceGenome; import org.mskcc.cbio.portal.dao.DaoReferenceGenomeGene; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ReferenceGenome; import org.mskcc.cbio.portal.model.ReferenceGenomeGene; @@ -659,7 +659,7 @@ public static long[] calculateGeneLength(List loci) { } static void importSuppGeneData(File suppGeneFile, String referenceGenomeBuild) throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); FileReader reader = new FileReader(suppGeneFile); BufferedReader buf = new BufferedReader(reader); String line; @@ -724,9 +724,9 @@ public void run() { int numLines = FileUtil.getNumLines(geneFile); System.out.println(" --> total number of lines: " + numLines); ProgressMonitor.setMaxValue(numLines); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); ImportGeneData.importData(geneFile, (String)options.valueOf("genome-build")); - MySQLbulkLoader.flushAll(); //Gene and gene_alias should be updated before calculating gene length (gtf)! + ClickHouseBulkLoader.flushAll(); //Gene and gene_alias should be updated before calculating gene length (gtf)! } if(options.has("hgnc")) { @@ -735,9 +735,9 @@ public void run() { int numLines = FileUtil.getNumLines(geneFile); System.out.println(" --> total number of lines: " + numLines); ProgressMonitor.setMaxValue(numLines); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); ImportGeneData.importHGNCData(geneFile, (String)options.valueOf("genome-build")); - MySQLbulkLoader.flushAll(); //Gene and gene_alias should be updated before calculating gene length (gtf)! + ClickHouseBulkLoader.flushAll(); //Gene and gene_alias should be updated before calculating gene length (gtf)! } if(options.has("supp-genes")) { @@ -772,7 +772,7 @@ public void run() { ImportGeneData.importGeneLength(lociFile, (String)options.valueOf("genome-build"), species, options.has("genes")); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); System.err.println("Done. Restart tomcat to make sure the cache is replaced with the new data."); } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java index 8da2c148..48a2db40 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java @@ -38,6 +38,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoGenePanel; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; @@ -158,6 +159,9 @@ public void importData() throws Exception { // Loop over gene panel matrix and load into database ProgressMonitor.setCurrentMessage("Loading gene panel profile matrix data to database.."); String row; + Set sampleProfileTuples = new HashSet<>(); + ClickHouseBulkLoader.bulkLoadOn(); + ProgressMonitor.logWarning("Reading gene panel profile file line by line."); while((row = buff.readLine()) != null) { List row_data = new LinkedList<>(Arrays.asList(row.split("\t"))); @@ -170,7 +174,6 @@ public void importData() throws Exception { row_data.remove((int)sampleIdIndex); - Set sampleProfileTuples = new HashSet<>(); // Loop over the values in the row for (int i = 0; i < row_data.size(); i++) { String genePanelName = row_data.get(i); @@ -188,20 +191,24 @@ public void importData() throws Exception { sampleProfileTuples.add(new DaoSampleProfile.SampleProfileTuple(geneticProfileId, sampleInternalId, genePanelId)); } - - DaoSampleProfile.upsertSampleToProfileMapping(sampleProfileTuples); } + ProgressMonitor.logWarning("Upserting sample to profile mappings into database."); + DaoSampleProfile.upsertSampleToProfileMapping(sampleProfileTuples); // update mutation counts with the latest sequencing information after the sample profile upsert // deals with issue where mutation counts are missing due to sample profile not yet updated at the time of // the first mutation count calculation during the mutation import ProgressMonitor.setCurrentMessage("Updating mutation counts in database.."); + ProgressMonitor.logWarning("Updating mutation counts for genetic profiles."); for (int i = 0; i < profileIds.size(); i++) { GeneticProfile geneticProfile = DaoGeneticProfile.getGeneticProfileById(profileIds.get(i)); if (geneticProfile.getGeneticAlterationType() == GeneticAlterationType.MUTATION_EXTENDED) { DaoMutation.createMutationCountClinicalData(geneticProfile); } } + ProgressMonitor.logWarning("Flushing ClickHouse bulk loader."); + ClickHouseBulkLoader.flushAll(); + ProgressMonitor.logWarning("Finished updating mutation counts for genetic profiles."); } private List getProfilesLine(BufferedReader buff) throws Exception { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java index 26650862..0c629364 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java @@ -249,8 +249,6 @@ public static void importData(File dataFile, GeneticAlterationType geneticAltera reader.close(); ProgressMonitor.setCurrentMessage("Finished loading generic assay.\n"); - - return; } // returns index for ENTITY_STABLE_ID column diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayPatientLevelData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayPatientLevelData.java index a7ce9b73..18ed11ee 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayPatientLevelData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayPatientLevelData.java @@ -35,6 +35,8 @@ import java.io.*; import java.util.*; import java.util.stream.*; + +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneticAlteration; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; @@ -42,7 +44,6 @@ import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; @@ -165,8 +166,8 @@ public void importData() throws IOException, DaoException { line = buf.readLine(); } - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); } if (entriesSkipped > 0) { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenesetData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenesetData.java index f9e91751..54102e61 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenesetData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenesetData.java @@ -40,9 +40,9 @@ import java.io.*; import java.util.*; import joptsimple.*; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoGeneset; import org.mskcc.cbio.portal.dao.DaoInfo; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.Geneset; import org.mskcc.cbio.portal.util.ProgressMonitor; @@ -256,7 +256,7 @@ public static int importData(File genesetFile, boolean updateInfo, boolean newV line = buf.readLine(); } //flush bulkloader to commit geneset genes: - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); // close file reader.close(); diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportMicroRNAIDs.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportMicroRNAIDs.java index 45e1fb10..6550bf57 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportMicroRNAIDs.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportMicroRNAIDs.java @@ -34,9 +34,10 @@ import java.io.*; import java.util.*; + +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; @@ -47,7 +48,7 @@ public class ImportMicroRNAIDs { public static void importData(File geneFile) throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); FileReader reader = new FileReader(geneFile); BufferedReader buf = new BufferedReader(reader); String line = buf.readLine(); // skip first line diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportProfileData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportProfileData.java index 3d6aa36b..92802aef 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportProfileData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportProfileData.java @@ -36,7 +36,6 @@ import java.util.*; import joptsimple.OptionSet; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; -import org.mskcc.cbio.portal.dao.DaoGeneticAlteration; import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.shared.MolecularProfileDataType; @@ -54,7 +53,6 @@ public class ImportProfileData extends ConsoleRunnable { public void run() { DaoGeneOptimized daoGene; - DaoGeneticAlteration daoGeneticAlteration; daoGene = DaoGeneOptimized.getInstance(); try { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportReferenceGenome.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportReferenceGenome.java index 09ae5c7b..b66dab76 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportReferenceGenome.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportReferenceGenome.java @@ -26,9 +26,9 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.commons.lang3.StringUtils; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoReferenceGenome; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.ReferenceGenome; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.FileUtil; @@ -141,11 +141,11 @@ public void run() { numLines = FileUtil.getNumLines(referenceFile); System.out.println(" --> total number of lines: " + numLines); ProgressMonitor.setMaxValue(numLines); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); ImportReferenceGenome.importData(referenceFile); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); System.err.println("Done. Restart tomcat to make sure the cache is replaced with the new data."); } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceData.java index 1d62f752..559a2104 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceData.java @@ -8,12 +8,12 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.commons.collections4.map.MultiKeyMap; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoResourceData; import org.mskcc.cbio.portal.dao.DaoResourceDefinition; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.ResourceDefinition; @@ -27,12 +27,14 @@ public class ImportResourceData extends ConsoleRunnable { public static final String DELIMITER = "\t"; - public static final String METADATA_PREFIX = "#"; + public static final String COMMENT_PREFIX = "#"; public static final String SAMPLE_ID_COLUMN_NAME = "SAMPLE_ID"; public static final String PATIENT_ID_COLUMN_NAME = "PATIENT_ID"; public static final String RESOURCE_ID_COLUMN_NAME = "RESOURCE_ID"; public static final String URL_COLUMN_NAME = "URL"; public static final String SAMPLE_TYPE_COLUMN_NAME = "SAMPLE_TYPE"; + public static final String METADATA_COLUMN_NAME = "METADATA"; + public static final String GROUP_PATH_COLUMN_NAME = "GROUP_PATH"; private int numSampleSpecificResourcesAdded = 0; private int numPatientSpecificResourcesAdded = 0; private int numStudySpecificResourcesAdded = 0; @@ -59,10 +61,10 @@ public void importData() throws Exception { // code has to be added to check whether // a resource data update should be // perform instead of an insert - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); if (relaxed) { - MySQLbulkLoader.relaxedModeOn(); + ClickHouseBulkLoader.relaxedModeOn(); } FileReader reader = new FileReader(resourceDataFile); @@ -99,9 +101,9 @@ public void importData() throws Exception { importData(buff, resources, headerIndexMap); buff.close(); - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); - MySQLbulkLoader.relaxedModeOff(); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.relaxedModeOff(); } } @@ -136,7 +138,7 @@ private void importData(BufferedReader buff, List resources, } private boolean skipLine(String line) { - return (line.isEmpty() || line.substring(0, 1).equals(METADATA_PREFIX)); + return (line.isEmpty() || line.substring(0, 1).equals(COMMENT_PREFIX)); } /** @@ -216,24 +218,29 @@ private boolean addDatum(String[] fields, List resources, Mu } } + int metadataIndex = findMetadataColumn(headerIndexMap); + int groupPathIndex = findGroupPathColumn(headerIndexMap); + String metadata = (metadataIndex >= 0 && !MissingValues.has(fields[metadataIndex])) ? fields[metadataIndex] : null; + String groupPath = (groupPathIndex >= 0 && !MissingValues.has(fields[groupPathIndex])) ? fields[groupPathIndex] : null; + // if the resource id or url matches one of the missing values, skip this resource: if ((resourceIdIndex != -1 && MissingValues.has(fields[resourceIdIndex])) || (urlIndex != -1 && MissingValues.has(fields[urlIndex]))) { numEmptyResourcesSkipped++; } else { // if patient_id column exists and resource type is patient if (getResourceType() == ResourceType.PATIENT && internalPatientId != -1) { - validateAddDatum(internalPatientId, stablePatientId, fields[resourceIdIndex], fields[urlIndex], - ResourceType.PATIENT, patientResourceIdSet, resourceMap); - } + validateAddDatum(internalPatientId, stablePatientId, fields[resourceIdIndex], fields[urlIndex], + ResourceType.PATIENT, patientResourceIdSet, resourceMap, metadata, groupPath); + } // if sample_id column exists and resource type is sample else if (getResourceType() == ResourceType.SAMPLE && internalSampleId != -1) { - validateAddDatum(internalSampleId, stableSampleId, fields[resourceIdIndex], fields[urlIndex], - ResourceType.SAMPLE, sampleResourceIdSet, resourceMap); + validateAddDatum(internalSampleId, stableSampleId, fields[resourceIdIndex], fields[urlIndex], + ResourceType.SAMPLE, sampleResourceIdSet, resourceMap, metadata, groupPath); } // if resource type is study else if (getResourceType() == ResourceType.STUDY) { - validateAddDatum(cancerStudy.getInternalId(), cancerStudy.getCancerStudyStableId(), fields[resourceIdIndex], fields[urlIndex], - ResourceType.STUDY, studyResourceIdSet, resourceMap); + validateAddDatum(cancerStudy.getInternalId(), cancerStudy.getCancerStudyStableId(), fields[resourceIdIndex], fields[urlIndex], + ResourceType.STUDY, studyResourceIdSet, resourceMap, metadata, groupPath); } } return true; @@ -267,6 +274,14 @@ private int findSampleTypeColumn(Map headerIndexMap) { return findColumnIndexInHeaders(SAMPLE_TYPE_COLUMN_NAME, headerIndexMap); } + private int findMetadataColumn(Map headerIndexMap) { + return findColumnIndexInHeaders(METADATA_COLUMN_NAME, headerIndexMap); + } + + private int findGroupPathColumn(Map headerIndexMap) { + return findColumnIndexInHeaders(GROUP_PATH_COLUMN_NAME, headerIndexMap); + } + private int findColumnIndexInHeaders(String columnHeader, Map headerIndexMap) { return headerIndexMap.getOrDefault(columnHeader, -1); } @@ -324,7 +339,7 @@ private boolean validSampleId(String sampleId) { return (sampleId != null && !sampleId.isEmpty()); } - private void validateAddDatum(int internalId, String stableId, String resourceId, String resourceURL, ResourceType resourceType, Set resourceSet, MultiKeyMap resourceMap) throws Exception { + private void validateAddDatum(int internalId, String stableId, String resourceId, String resourceURL, ResourceType resourceType, Set resourceSet, MultiKeyMap resourceMap, String metadata, String groupPath) throws Exception { // throw exception if resource definition is not exist in the database if (!resourceSet.contains(resourceId)) { throw new RuntimeException("Error: " + resourceType.toString().toLowerCase() + " " + stableId @@ -334,7 +349,7 @@ private void validateAddDatum(int internalId, String stableId, String resourceId // The resourceMap makes sure a pair of (internalId/resource_id/url) is unique // added to the DB if there are no duplicates, if (!resourceMap.containsKey(internalId, resourceId, resourceURL)) { - addDatum(internalId, resourceId, resourceURL,resourceType); + addDatum(internalId, resourceId, resourceURL, resourceType, metadata, groupPath); resourceMap.put(internalId, resourceId, resourceURL, resourceURL); } // handle duplicates @@ -351,18 +366,19 @@ else if (!resourceMap.get(internalId, resourceId, resourceURL).equals(resourceUR } // add datum for patient, sample and study resources - private void addDatum(int internalId, String resourceId, String resourceURL, ResourceType resourceType) throws Exception { + private void addDatum(int internalId, String resourceId, String resourceURL, ResourceType resourceType, String metadata, String groupPath) throws Exception { // if bulk loading is ever turned off, we need to check if - // resource value exists and if so, perfom an update + // resource value exists and if so, perform an update + int cancerStudyId = cancerStudy.getInternalId(); if (resourceType.equals(ResourceType.PATIENT)) { numPatientSpecificResourcesAdded++; - DaoResourceData.addPatientDatum(internalId, resourceId, resourceURL); + DaoResourceData.addPatientDatum(internalId, cancerStudyId, resourceId, resourceId, resourceURL, metadata, groupPath); } else if (resourceType.equals(ResourceType.SAMPLE)) { numSampleSpecificResourcesAdded++; - DaoResourceData.addSampleDatum(internalId, resourceId, resourceURL); + DaoResourceData.addSampleDatum(internalId, cancerStudyId, resourceId, resourceId, resourceURL, metadata, groupPath); } else { numStudySpecificResourcesAdded++; - DaoResourceData.addStudyDatum(internalId, resourceId, resourceURL); + DaoResourceData.addStudyDatum(internalId, cancerStudyId, resourceId, resourceId, resourceURL, metadata, groupPath); } } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceDefinition.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceDefinition.java index 54a62023..26a6d2b6 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceDefinition.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportResourceDefinition.java @@ -6,10 +6,10 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoResourceDefinition; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ResourceDefinition; import org.mskcc.cbio.portal.model.shared.ResourceType; @@ -47,10 +47,10 @@ public void importData() throws Exception { // code has to be added to check whether // a resource data update should be // perform instead of an insert - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); if (relaxed) { - MySQLbulkLoader.relaxedModeOn(); + ClickHouseBulkLoader.relaxedModeOn(); } FileReader reader = new FileReader(resourceDataFile); @@ -143,9 +143,9 @@ public void importData() throws Exception { } buff.close(); - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); - MySQLbulkLoader.relaxedModeOff(); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.relaxedModeOff(); } } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportStructuralVariantData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportStructuralVariantData.java index b2f9e614..aef89c0e 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportStructuralVariantData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportStructuralVariantData.java @@ -26,13 +26,13 @@ import java.io.*; import java.util.*; import org.mskcc.cbio.maf.TabDelimitedFileUtil; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.dao.DaoStructuralVariant; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.Sample; @@ -190,13 +190,13 @@ public void importData() throws IOException, DaoException { } } - DaoSampleProfile.upsertSampleToProfileMapping(sampleIds, geneticProfileId, genePanelId); if (isIncrementalUpdateMode) { DaoStructuralVariant.deleteStructuralVariants(geneticProfileId, sampleIds); } + DaoSampleProfile.upsertSampleToProfileMapping(sampleIds, geneticProfileId, genePanelId); buf.close(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } private CanonicalGene setCanonicalGene(long siteEntrezGeneId, String siteHugoSymbol, DaoGeneOptimized daoGene) { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java index e4f1bc67..203d047f 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java @@ -38,6 +38,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCnaEvent; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; @@ -45,8 +46,6 @@ import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; -import org.mskcc.cbio.portal.dao.JdbcUtil; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.CnaEvent; import org.mskcc.cbio.portal.model.Geneset; @@ -171,51 +170,34 @@ public ImportTabDelimData( * Import the Copy Number Alteration, mRNA Expression, protein RPPA, GSVA or generic_assay data * */ - public void importData() { - JdbcUtil.getTransactionTemplate().execute(status -> { - try { - doImportData(); - } catch (Throwable e) { - status.setRollbackOnly(); - throw new RuntimeException(e); - } - return null; - }); - } - private void doImportData() throws IOException, DaoException { - try { - this.numLines = FileUtil.getNumLines(dataFile); - } catch (IOException e) { - throw new RuntimeException(e); - } + public void importData() throws Exception { + this.numLines = FileUtil.getNumLines(dataFile); ProgressMonitor.setMaxValue(numLines); - FileReader reader = new FileReader(dataFile); - BufferedReader buf = new BufferedReader(reader); - String headerLine = buf.readLine(); - String[] headerParts = TsvUtil.splitTsvLine(headerLine); - - //Whether data regards CNA or RPPA: - boolean isDiscretizedCnaProfile = geneticProfile != null - && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.COPY_NUMBER_ALTERATION - && geneticProfile.showProfileInAnalysisTab(); - boolean isRppaProfile = geneticProfile != null - && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.PROTEIN_LEVEL - && "Composite.Element.Ref".equalsIgnoreCase(headerParts[0]); - boolean isGsvaProfile = geneticProfile != null - && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.GENESET_SCORE - && headerParts[0].equalsIgnoreCase("geneset_id"); - boolean isGenericAssayProfile = geneticProfile != null - && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.GENERIC_ASSAY - && headerParts[0].equalsIgnoreCase("ENTITY_STABLE_ID"); - - long typesDetected = List.of(isDiscretizedCnaProfile, isRppaProfile, isGsvaProfile, isGenericAssayProfile).stream().filter(Boolean::booleanValue).count(); - if (typesDetected > 1) { - throw new IllegalStateException("More then one data type is detected."); - } - - int numRecordsToAdd = 0; - int samplesSkipped = 0; - try { + try (FileReader reader = new FileReader(dataFile); + BufferedReader buf = new BufferedReader(reader)) { + String headerLine = buf.readLine(); + String[] headerParts = TsvUtil.splitTsvLine(headerLine); + //Whether data regards CNA or RPPA: + boolean isDiscretizedCnaProfile = geneticProfile != null + && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.COPY_NUMBER_ALTERATION + && geneticProfile.showProfileInAnalysisTab(); + boolean isRppaProfile = geneticProfile != null + && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.PROTEIN_LEVEL + && "Composite.Element.Ref".equalsIgnoreCase(headerParts[0]); + boolean isGsvaProfile = geneticProfile != null + && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.GENESET_SCORE + && headerParts[0].equalsIgnoreCase("geneset_id"); + boolean isGenericAssayProfile = geneticProfile != null + && geneticProfile.getGeneticAlterationType() == GeneticAlterationType.GENERIC_ASSAY + && headerParts[0].equalsIgnoreCase("ENTITY_STABLE_ID"); + + long typesDetected = List.of(isDiscretizedCnaProfile, isRppaProfile, isGsvaProfile, isGenericAssayProfile).stream().filter(Boolean::booleanValue).count(); + if (typesDetected > 1) { + throw new IllegalStateException("More then one data type is detected."); + } + + int numRecordsToAdd = 0; + int samplesSkipped = 0; int hugoSymbolIndex = getHugoSymbolIndex(headerParts); int entrezGeneIdIndex = getEntrezGeneIdIndex(headerParts); int rppaGeneRefIndex = getRppaGeneRefIndex(headerParts); @@ -296,7 +278,7 @@ private void doImportData() throws IOException, DaoException { Set existingCnaEvents = new HashSet<>(); if (isDiscretizedCnaProfile) { existingCnaEvents.addAll(DaoCnaEvent.getAllCnaEvents()); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } // load entities map from database @@ -375,8 +357,8 @@ private void doImportData() throws IOException, DaoException { } DaoSampleProfile.upsertSampleToProfileMapping(orderedSampleList, geneticProfileId, genePanelId); geneticAlterationImporter.finalize(); - if (MySQLbulkLoader.isBulkLoad()) { - MySQLbulkLoader.flushAll(); + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); } if (isRppaProfile) { @@ -391,9 +373,6 @@ private void doImportData() throws IOException, DaoException { " to the database!"); } } - finally { - buf.close(); - } } private Map, Map> readPdAnnotations(File pdAnnotationsFile) { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTimelineData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTimelineData.java index 7f2c6114..6e2d942d 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTimelineData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTimelineData.java @@ -35,10 +35,10 @@ import java.io.*; import java.util.*; import joptsimple.OptionSet; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoClinicalEvent; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoPatient; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.ClinicalEvent; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.util.ConsoleUtil; @@ -52,7 +52,7 @@ public class ImportTimelineData extends ConsoleRunnable { private static void importData(String dataFile, int cancerStudyId, boolean overwriteExisting) throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); ProgressMonitor.setCurrentMessage("Reading file " + dataFile); FileReader reader = new FileReader(dataFile); @@ -115,7 +115,7 @@ private static void importData(String dataFile, int cancerStudyId, boolean overw DaoClinicalEvent.addClinicalEvent(event); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } finally { buff.close(); diff --git a/src/main/java/org/mskcc/cbio/portal/util/ConsoleUtil.java b/src/main/java/org/mskcc/cbio/portal/util/ConsoleUtil.java index 7e84143e..264ea611 100644 --- a/src/main/java/org/mskcc/cbio/portal/util/ConsoleUtil.java +++ b/src/main/java/org/mskcc/cbio/portal/util/ConsoleUtil.java @@ -39,7 +39,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.scripts.UsageException; /** @@ -167,9 +167,9 @@ public static OptionSet parseStandardDataAndMetaOptions(String[] args, String de if( options.has( "loadMode" ) ){ String actionArg = (String) options.valueOf( "loadMode" ); if (actionArg.equalsIgnoreCase("directLoad")) { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); } else if (actionArg.equalsIgnoreCase( "bulkLoad" )) { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } else { throw new UsageException(progName, description, parser, "Error: unknown loadMode action: " + actionArg); @@ -279,9 +279,9 @@ public static OptionSet parseStandardDataAndMetaUpdateOptions(String[] args, Str if( options.has( "loadMode" ) ){ String actionArg = (String) options.valueOf( "loadMode" ); if (actionArg.equalsIgnoreCase("directLoad")) { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); } else if (actionArg.equalsIgnoreCase( "bulkLoad" )) { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } else { throw new UsageException(progName, description, parser, "Error: unknown loadMode action: " + actionArg); diff --git a/src/main/java/org/mskcc/cbio/portal/util/MutSigReader.java b/src/main/java/org/mskcc/cbio/portal/util/MutSigReader.java index 5a2f5a5e..02ad2df7 100644 --- a/src/main/java/org/mskcc/cbio/portal/util/MutSigReader.java +++ b/src/main/java/org/mskcc/cbio/portal/util/MutSigReader.java @@ -33,9 +33,10 @@ package org.mskcc.cbio.portal.util; import java.io.*; + +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.MutSig; import static org.mskcc.cbio.portal.dao.DaoMutSig.addMutSig; @@ -62,7 +63,7 @@ public class MutSigReader { */ public static int loadMutSig(int internalId, File mutSigFile) throws IOException, DaoException { int loadedMutSigs = 0; - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); FileReader reader = new FileReader(mutSigFile); BufferedReader buf = new BufferedReader(reader); try { diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/IntegrationTestBase.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/IntegrationTestBase.java index 8f6503cf..7060e9b1 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/IntegrationTestBase.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/IntegrationTestBase.java @@ -13,56 +13,74 @@ import java.sql.ResultSet; import java.sql.Statement; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.testcontainers.containers.MySQLContainer; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.mskcc.cbio.portal.dao.ClickHouseAutoIncrement; +import org.mskcc.cbio.portal.dao.DaoCancerStudy; +import org.springframework.test.annotation.DirtiesContext; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.Container; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public abstract class IntegrationTestBase { private static final Object LOCK = new Object(); private static final String DB_NAME = "cgds_test"; private static final String DB_USER = "cbio_user"; private static final String DB_PASSWORD = "somepassword"; - private static final String ROOT_PASSWORD = "root"; - private static final String MYSQL_IMAGE = "mysql:5.7"; + private static final int CLICKHOUSE_HTTP_PORT = 8123; + private static final String CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server:24.3"; private static final String SKIP_ENV = "CBIOPORTAL_TEST_DB_SKIP"; private static final String TARGET_DIR = "target/test-db"; private static final String VERSION_FILE = "cbioportal.version"; - private static final String CGDS_SQL_FILE = "cgds.sql"; + private static final String CGDS_SQL_FILE = "clickhouse_cgds.sql"; + private static final String SEED_SQL_FILE = "seed_mini.sql"; private static final String CGDS_URL_TEMPLATE = - "https://raw.githubusercontent.com/cBioPortal/cbioportal/%s/src/main/resources/db-scripts/cgds.sql"; + "https://raw.githubusercontent.com/cBioPortal/cbioportal/%s/src/test/resources/clickhouse_cgds.sql"; private static final Pattern CBIOPORTAL_VERSION_PATTERN = Pattern.compile("\\s*([^<]+)\\s*"); - private static MySQLContainer container; + private static ClickHouseContainer container; static { Runtime.getRuntime().addShutdownHook(new Thread(IntegrationTestBase::stopContainer)); - ensureStarted(); } - protected static void ensureStarted() { - getOrStartContainer(); + @BeforeClass + public static void setupTestDatabase() { + startContainer(); } - protected static void restartTestDb() { - synchronized (LOCK) { - if (shouldSkip()) { - return; - } - stopContainer(); - container = startContainer(); - } + @Before + public void reCache() { + DaoCancerStudy.reCacheAll(); } - private static MySQLContainer getOrStartContainer() { + @After + public void resetDatabase() { + recreateContainer(); + ClickHouseAutoIncrement.resetCounters(); + } + + @AfterClass + public static void teardownTestDatabase() { + stopContainer(); + } + + private static void startContainer() { synchronized (LOCK) { if (container == null && !shouldSkip()) { - container = startContainer(); + container = startAndInitializeContainer(); } - return container; } } @@ -75,6 +93,11 @@ private static void stopContainer() { } } + private static void recreateContainer() { + stopContainer(); + startContainer(); + } + private static boolean shouldSkip() { String value = System.getenv(SKIP_ENV); if (value == null) { @@ -87,29 +110,25 @@ private static boolean shouldSkip() { return "1".equals(normalized) || "true".equals(normalized); } - private static MySQLContainer startContainer() { + private static ClickHouseContainer startAndInitializeContainer() { Path cgdsPath = ensureCgdsSql(); - MySQLContainer mysql = new MySQLContainer<>(DockerImageName.parse(MYSQL_IMAGE)) - .withDatabaseName(DB_NAME) + Path seedPath = ensureSeedSql(); + ClickHouseContainer clickhouse = new ClickHouseContainer(DockerImageName.parse(CLICKHOUSE_IMAGE)) .withUsername(DB_USER) .withPassword(DB_PASSWORD) - .withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD) - .withStartupTimeout(Duration.ofMinutes(5)) - .withCopyFileToContainer( - MountableFile.forClasspathResource("seed_mini.sql"), - "/docker-entrypoint-initdb.d/seed.sql") - .withCopyFileToContainer( - MountableFile.forHostPath(cgdsPath), - "/docker-entrypoint-initdb.d/cgds.sql"); - mysql.start(); - setSystemProperties(mysql); - waitForSchema(mysql); - return mysql; - } - - private static void setSystemProperties(MySQLContainer mysql) { - String host = mysql.getHost(); - String port = Integer.toString(mysql.getMappedPort(3306)); + .withEnv("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1") + .withStartupTimeout(Duration.ofMinutes(5)); + clickhouse.start(); + applySchema(clickhouse, cgdsPath); + applySeed(clickhouse, seedPath); + setSystemProperties(clickhouse); + waitForSchema(clickhouse); + return clickhouse; + } + + private static void setSystemProperties(ClickHouseContainer clickhouse) { + String host = clickhouse.getHost(); + String port = Integer.toString(clickhouse.getMappedPort(CLICKHOUSE_HTTP_PORT)); String jdbcUrl = buildJdbcUrl(host, port, DB_NAME); System.setProperty("db.test.host", host); @@ -121,17 +140,24 @@ private static void setSystemProperties(MySQLContainer mysql) { System.setProperty("spring.datasource.url", jdbcUrl); System.setProperty("spring.datasource.username", DB_USER); System.setProperty("spring.datasource.password", DB_PASSWORD); - System.setProperty("spring.datasource.driver-class-name", "com.mysql.jdbc.Driver"); + System.setProperty("spring.datasource.driver-class-name", "com.clickhouse.jdbc.ClickHouseDriver"); } - private static void waitForSchema(MySQLContainer mysql) { - String jdbcUrl = buildJdbcUrl(mysql.getHost(), Integer.toString(mysql.getMappedPort(3306)), DB_NAME); + private static void waitForSchema(ClickHouseContainer clickhouse) { + String jdbcUrl = buildJdbcUrl( + clickhouse.getHost(), + Integer.toString(clickhouse.getMappedPort(CLICKHOUSE_HTTP_PORT)), + DB_NAME); long deadline = System.currentTimeMillis() + Duration.ofMinutes(5).toMillis(); + String schemaQuery = String.format("SELECT count(*) FROM %s.info", DB_NAME); while (System.currentTimeMillis() < deadline) { try (Connection connection = DriverManager.getConnection(jdbcUrl, DB_USER, DB_PASSWORD); Statement statement = connection.createStatement(); - ResultSet result = statement.executeQuery("SELECT 1 FROM cancer_study LIMIT 1")) { + ResultSet result = statement.executeQuery(schemaQuery)) { if (result.next()) { + if (result.getInt(1) < 1) { + continue; + } return; } } catch (Exception ex) { @@ -148,13 +174,76 @@ private static void waitForSchema(MySQLContainer mysql) { private static String buildJdbcUrl(String host, String port, String database) { return String.format( - "jdbc:mysql://%s:%s/%s?allowPublicKeyRetrieval=true&allowLoadLocalInfile=true&useSSL=false", + "jdbc:clickhouse://%s:%s/%s", host, port, database); } + private static void applySchema(ClickHouseContainer clickhouse, Path schemaPath) { + String containerSchemaPath = "/tmp/" + CGDS_SQL_FILE; + clickhouse.copyFileToContainer(MountableFile.forHostPath(schemaPath), containerSchemaPath); + + executeClickHouseQuery(clickhouse, "CREATE DATABASE IF NOT EXISTS " + DB_NAME); + + applySqlFile(clickhouse, containerSchemaPath); + } + + private static void applySeed(ClickHouseContainer clickhouse, Path seedPath) { + String containerSeedPath = "/tmp/" + SEED_SQL_FILE; + clickhouse.copyFileToContainer(MountableFile.forHostPath(seedPath), containerSeedPath); + + applySqlFile(clickhouse, containerSeedPath); + } + + private static void applySqlFile(ClickHouseContainer clickhouse, String containerSeedPath) { + List command = baseClickHouseCommand(); + command.add("--database"); + command.add(DB_NAME); + command.add("--multiquery"); + command.add("--queries-file"); + command.add(containerSeedPath); + executeClickHouseCommand(clickhouse, command); + } + + private static void executeClickHouseQuery(ClickHouseContainer clickhouse, String query) { + List command = baseClickHouseCommand(); + command.add("--query"); + command.add(query); + executeClickHouseCommand(clickhouse, command); + } + + private static List baseClickHouseCommand() { + List command = new ArrayList<>(); + command.add("clickhouse-client"); + command.add("--user"); + command.add(DB_USER); + command.add("--password"); + command.add(DB_PASSWORD); + return command; + } + + private static void executeClickHouseCommand(ClickHouseContainer clickhouse, List command) { + try { + Container.ExecResult result = clickhouse.execInContainer(command.toArray(new String[0])); + if (result.getExitCode() != 0) { + String error = result.getStderr(); + if (error == null || error.isBlank()) { + error = result.getStdout(); + } + throw new IllegalStateException("ClickHouse command failed: " + error); + } + } catch (IOException ex) { + throw new IllegalStateException("Failed to execute ClickHouse command", ex); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while executing ClickHouse command", ex); + } + } + private static Path ensureCgdsSql() { + //FIXME Uncomment when cbioportal contains the complete clickhouse_cgds.sql for the current version + /* String cbioportalVersion = readCbioportalVersion(); Path targetDir = Paths.get(System.getProperty("user.dir"), TARGET_DIR); Path cgdsPath = targetDir.resolve(CGDS_SQL_FILE); @@ -167,11 +256,21 @@ private static Path ensureCgdsSql() { Files.writeString(versionPath, cbioportalVersion, StandardCharsets.UTF_8); } } catch (IOException ex) { - throw new IllegalStateException("Failed to prepare cgds.sql for integration tests", ex); + throw new IllegalStateException("Failed to prepare clickhouse_cgds.sql for integration tests", ex); } + */ + Path cgdsPath = Paths.get(System.getProperty("user.dir"), "src", "test", "resources", CGDS_SQL_FILE); return cgdsPath; } + private static Path ensureSeedSql() { + Path seedPath = Paths.get(System.getProperty("user.dir"), "src", "test", "resources", SEED_SQL_FILE); + if (!Files.exists(seedPath)) { + throw new IllegalStateException("seed_mini.sql not found at " + seedPath); + } + return seedPath; + } + private static String readCbioportalVersion() { Path pomPath = Paths.get(System.getProperty("user.dir"), "pom.xml"); try { @@ -192,7 +291,7 @@ private static String readFile(Path path) throws IOException { private static void downloadCgdsSql(String version, Path destination) throws IOException { String url = String.format(CGDS_URL_TEMPLATE, version); - System.out.println("Downloading cgds.sql for " + version + "..."); + System.out.println("Downloading clickhouse_cgds.sql for " + version + "..."); try (InputStream input = new URL(url).openStream()) { Files.copy(input, destination, StandardCopyOption.REPLACE_EXISTING); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoCancerStudy.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoCancerStudy.java index be86c7f5..77fa3a93 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoCancerStudy.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoCancerStudy.java @@ -41,10 +41,8 @@ import org.mskcc.cbio.portal.dao.DaoTypeOfCancer; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ReferenceGenome; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -57,8 +55,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoCancerStudy extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoClinicalAttribute.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoClinicalAttribute.java index 49e983e6..31c1e1b0 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoClinicalAttribute.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoClinicalAttribute.java @@ -37,18 +37,14 @@ import org.mskcc.cbio.portal.dao.DaoClinicalAttributeMeta; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.model.ClinicalAttribute; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoClinicalAttribute extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGene.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGene.java index f6fa7936..4de24a63 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGene.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGene.java @@ -35,14 +35,12 @@ import java.util.*; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -51,8 +49,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoGene extends IntegrationTestBase { /** @@ -63,8 +59,8 @@ public class TestDaoGene extends IntegrationTestBase { public void testAddExistingGene() throws DaoException { // save bulkload setting before turning off - boolean isBulkLoad = MySQLbulkLoader.isBulkLoad(); - MySQLbulkLoader.bulkLoadOff(); + boolean isBulkLoad = ClickHouseBulkLoader.isBulkLoad(); + ClickHouseBulkLoader.bulkLoadOff(); // Add BRCA1 and BRCA2 Genes CanonicalGene gene = new CanonicalGene(672, "BRCA1", @@ -75,7 +71,7 @@ public void testAddExistingGene() throws DaoException { // restore bulk setting if (isBulkLoad) { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } } @@ -87,8 +83,8 @@ public void testAddExistingGene() throws DaoException { public void testAddNewGene() throws DaoException { // save bulkload setting before turning off - boolean isBulkLoad = MySQLbulkLoader.isBulkLoad(); - MySQLbulkLoader.bulkLoadOff(); + boolean isBulkLoad = ClickHouseBulkLoader.isBulkLoad(); + ClickHouseBulkLoader.bulkLoadOff(); // Add BRCA1 and BRCA2 Genes CanonicalGene gene = new CanonicalGene(1956, "EGFR", @@ -99,7 +95,7 @@ public void testAddNewGene() throws DaoException { // restore bulk setting if (isBulkLoad) { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGenePanel.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGenePanel.java index 388d74a6..b96e666e 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGenePanel.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGenePanel.java @@ -40,10 +40,8 @@ import org.mskcc.cbio.portal.dao.DaoGenePanel; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.GenePanel; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -53,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoGenePanel extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticAlteration.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticAlteration.java index bc9fc23d..eba4b713 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticAlteration.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticAlteration.java @@ -36,6 +36,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneticAlteration; @@ -43,15 +44,12 @@ import org.mskcc.cbio.portal.dao.DaoGeneticProfileSamples; import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -60,8 +58,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoGeneticAlteration extends IntegrationTestBase { CancerStudy study; @@ -91,17 +87,17 @@ public void setUp() throws DaoException { @Test public void testDaoGeneticAlterationBulkOn() throws DaoException { - // test with MySQLbulkLoader.isBulkLoad() + // test with ClickHouseBulkLoader.isBulkLoad() runTheTest(); } @Test public void testDaoGeneticAlterationBulkOff() throws DaoException { - // test without MySQLbulkLoader.isBulkLoad() - MySQLbulkLoader.bulkLoadOff(); + // test without ClickHouseBulkLoader.isBulkLoad() + ClickHouseBulkLoader.bulkLoadOff(); runTheTest(); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); } private void runTheTest() throws DaoException{ @@ -118,8 +114,8 @@ private void runTheTest() throws DaoException{ assertEquals (1, numRows); // if bulkLoading, execute LOAD FILE - if( MySQLbulkLoader.isBulkLoad()){ - MySQLbulkLoader.flushAll(); + if( ClickHouseBulkLoader.isBulkLoad()){ + ClickHouseBulkLoader.flushAll(); } HashMap valueMap = dao.getGeneticAlterationMap(geneticProfileId, 672); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticProfile.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticProfile.java index d6900d8b..0f330540 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticProfile.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGeneticProfile.java @@ -41,10 +41,8 @@ import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -54,8 +52,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoGeneticProfile extends IntegrationTestBase { int studyId; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGistic.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGistic.java index fc716bc3..3728c5ce 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGistic.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoGistic.java @@ -42,18 +42,14 @@ import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.Gistic; import org.mskcc.cbio.portal.validate.validationException; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoGistic extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutSig.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutSig.java index 7cf7ea20..c01c4894 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutSig.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutSig.java @@ -40,10 +40,8 @@ import org.mskcc.cbio.portal.dao.DaoMutSig; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.MutSig; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -54,8 +52,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoMutSig extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutation.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutation.java index 1639c763..540214fb 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutation.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoMutation.java @@ -38,20 +38,18 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.dao.DaoMutation; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ExtendedMutation; import org.mskcc.cbio.portal.model.GeneticProfile; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -61,8 +59,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoMutation extends IntegrationTestBase { int geneticProfileId; @@ -84,7 +80,7 @@ public void setUp() throws DaoException { @Test public void testDaoMutation() throws DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); runTheTest(); } @@ -144,8 +140,8 @@ private void runTheTest() throws DaoException{ DaoMutation.addMutation(mutation,true); // if bulkLoading, execute LOAD FILE - if( MySQLbulkLoader.isBulkLoad()){ - MySQLbulkLoader.flushAll(); + if( ClickHouseBulkLoader.isBulkLoad()){ + ClickHouseBulkLoader.flushAll(); } ArrayList mutationList = DaoMutation.getMutations(geneticProfileId, 1, 321); validateMutation(mutationList.get(0)); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleList.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleList.java index c230b256..30067e4c 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleList.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleList.java @@ -46,10 +46,8 @@ import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.model.SampleList; import org.mskcc.cbio.portal.model.SampleListCategory; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -58,8 +56,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoSampleList extends IntegrationTestBase { CancerStudy study; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleProfile.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleProfile.java index 5014dc38..75364f39 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleProfile.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoSampleProfile.java @@ -49,10 +49,8 @@ import org.mskcc.cbio.portal.model.GenePanel; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -62,8 +60,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoSampleProfile extends IntegrationTestBase { CancerStudy study; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUser.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUser.java index 73582452..9f7a0eb9 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUser.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUser.java @@ -37,10 +37,8 @@ import org.junit.Test; import org.mskcc.cbio.portal.dao.DaoUser; import org.mskcc.cbio.portal.model.User; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; @@ -55,8 +53,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoUser extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUserAuthorities.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUserAuthorities.java index 243c5119..a7dcb111 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUserAuthorities.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/dao/TestDaoUserAuthorities.java @@ -38,10 +38,8 @@ import org.mskcc.cbio.portal.dao.DaoUserAuthorities; import org.mskcc.cbio.portal.model.User; import org.mskcc.cbio.portal.model.UserAuthorities; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -53,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestDaoUserAuthorities extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberAlterationImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberAlterationImport.java index 2b75391e..caea0f9c 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberAlterationImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberAlterationImport.java @@ -20,12 +20,8 @@ import java.io.*; import java.util.*; import java.util.stream.Collectors; -import org.junit.After; -import org.junit.Before; import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import org.junit.Test; -import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoCnaEvent; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGenePanel; @@ -38,11 +34,10 @@ import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportProfileData; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestContextManager; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.support.DefaultTransactionDefinition; -import org.springframework.transaction.TransactionStatus; + import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.mskcc.cbio.portal.integrationTest.incremental.GeneticAlterationsTestHelper.assertNoChange; @@ -54,7 +49,7 @@ * @author Ruslan Forostianov * @author Pieter Lukasse */ -@RunWith(Parameterized.class) +@RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) public class TestIncrementalCopyNumberAlterationImport extends IntegrationTestBase { @@ -65,8 +60,6 @@ public class TestIncrementalCopyNumberAlterationImport extends IntegrationTestBa final long absentGeneEntrezId = 472l; final Set noChangeEntrezIds = Set.of(10000l, 207l, 208l, 3265l, 3845l, 4893l, 672l, 673l, 675l); final Set beforeEntrezIds = new HashSet<>(noChangeEntrezIds); - private final String metaFile; - private final String dataFile; { beforeEntrezIds.add(absentGeneEntrezId); } @@ -81,24 +74,20 @@ public class TestIncrementalCopyNumberAlterationImport extends IntegrationTestBa final Set afterSampleIds = new HashSet<>(beforeSampleIds); { afterSampleIds.add(newSampleId); } - @Parameterized.Parameters(name = "{0}") - public static Collection primeNumbers() { - return Arrays.asList(new Object[][] { - { "meta_cna_discrete.txt", "data_cna_discrete.txt" }, - { "meta_cna_discrete_long.txt", "data_cna_discrete_long.txt" }, - }); + @Test + public void testDiscreteCNA_wide() throws DaoException { + testDiscreteCNA( "meta_cna_discrete.txt", "data_cna_discrete.txt"); } - public TestIncrementalCopyNumberAlterationImport(String metaFile, String dataFile) { - this.metaFile = metaFile; - this.dataFile = dataFile; + @Test + public void testDiscreteCNA_long() throws DaoException { + testDiscreteCNA( "meta_cna_discrete_long.txt", "data_cna_discrete_long.txt"); } /** * Test incremental upload of COPY_NUMBER_ALTERATION DISCRETE */ - @Test - public void testDiscreteCNA() throws DaoException { + private void testDiscreteCNA(String metaFileName, String dataFileName) throws DaoException { GeneticProfile discreteCNAProfile = DaoGeneticProfile.getGeneticProfileByStableId("study_tcga_pub_gistic"); assertNotNull(discreteCNAProfile); HashMap> beforeResult = DaoGeneticAlteration.getInstance().getGeneticAlterationMap(discreteCNAProfile.getGeneticProfileId(), null); @@ -129,8 +118,8 @@ public void testDiscreteCNA() throws DaoException { assertEquals(beforeCnaEventsSampleIds, beforeSampleIdToSampleCnaEvents.keySet()); File dataFolder = new File("src/test/resources/incremental/copy_number_alteration/"); - File metaFile = new File(dataFolder, this.metaFile); - File dataFile = new File(dataFolder, this.dataFile); + File metaFile = new File(dataFolder, metaFileName); + File dataFile = new File(dataFolder, dataFileName); /** * Test @@ -218,23 +207,4 @@ public void testDiscreteCNA() throws DaoException { DaoSampleProfile.getPanelId(sampleId, discreteCNAProfile.getGeneticProfileId())); } } - - private TestContextManager testContextManager; - - private PlatformTransactionManager transactionManager; - - private TransactionStatus transactionStatus; - @Before - public void before() throws Exception { - this.testContextManager = new TestContextManager(getClass()); - this.testContextManager.prepareTestInstance(this); - this.transactionManager = this.testContextManager.getTestContext().getApplicationContext().getBean(PlatformTransactionManager.class); - this.transactionStatus = transactionManager.getTransaction(new DefaultTransactionDefinition()); - DaoCancerStudy.reCacheAll(); - } - - @After - public void after() { - this.transactionManager.rollback(transactionStatus); - } } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberSegmentDataImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberSegmentDataImport.java index 7fef075c..48459536 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberSegmentDataImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalCopyNumberSegmentDataImport.java @@ -22,23 +22,21 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalData; import org.mskcc.cbio.portal.dao.DaoCopyNumberSegment; import org.mskcc.cbio.portal.dao.DaoCopyNumberSegmentFile; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ClinicalData; import org.mskcc.cbio.portal.model.CopyNumberSegment; import org.mskcc.cbio.portal.model.CopyNumberSegmentFile; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportCopyNumberSegmentData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -51,8 +49,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalCopyNumberSegmentDataImport extends IntegrationTestBase { /** @@ -71,7 +67,7 @@ public void testIncrementalUpload() throws DaoException { copyNumberSegmentFile.description = "test seg file description"; DaoCopyNumberSegmentFile.addCopyNumberSegmentFile(copyNumberSegmentFile); DaoClinicalData.addSampleDatum(segDataSample.getInternalId(), "FRACTION_GENOME_ALTERED", "TEST"); - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); CopyNumberSegment copyNumberSegment = new CopyNumberSegment( cancerStudy.getInternalId(), segDataSample.getInternalId(), @@ -82,7 +78,7 @@ public void testIncrementalUpload() throws DaoException { 0.01); copyNumberSegment.setSegId(1L); DaoCopyNumberSegment.addCopyNumberSegment(copyNumberSegment); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); File dataFolder = new File("src/test/resources/incremental/copy_number_alteration/"); File metaFile = new File(dataFolder, "meta_cna_seg.txt"); @@ -105,7 +101,7 @@ public void testIncrementalUpload() throws DaoException { List clinicalData = DaoClinicalData.getSampleData(cancerStudy.getInternalId(), Set.of(segSampleId)); ClinicalData fractionGenomeAltered = clinicalData.stream() .filter(cd -> "FRACTION_GENOME_ALTERED".equals(cd.getAttrId())).findFirst().get(); - assertEquals("0.0000", fractionGenomeAltered.getAttrVal()); + assertEquals("0.0", fractionGenomeAltered.getAttrVal()); } public static final String STUDY_ID = "study_tcga_pub"; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenePanelMatrixImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenePanelMatrixImport.java index 7e55507b..49971aac 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenePanelMatrixImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenePanelMatrixImport.java @@ -19,8 +19,6 @@ import java.io.*; import java.util.*; -import java.util.stream.Collectors; -import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; import org.mskcc.cbio.portal.dao.DaoCancerStudy; @@ -30,21 +28,15 @@ import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.model.CancerStudy; -import org.mskcc.cbio.portal.model.ExtendedMutation; import org.mskcc.cbio.portal.model.GenePanel; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap; -import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.mskcc.cbio.portal.dao.DaoMutation.getMutations; /** * Tests Incremental Import of Gene Panel Matrix Data. @@ -54,8 +46,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalGenePanelMatrixImport extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenericAssayImporter.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenericAssayImporter.java index facc3e9d..3946db9e 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenericAssayImporter.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGenericAssayImporter.java @@ -32,15 +32,12 @@ import org.mskcc.cbio.portal.model.GenePanel; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.mskcc.cbio.portal.integrationTest.incremental.GeneticAlterationsTestHelper.assertNoChange; import static org.mskcc.cbio.portal.integrationTest.incremental.GeneticAlterationsTestHelper.assertPriorDataState; @@ -55,8 +52,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalGenericAssayImporter extends IntegrationTestBase { // stable_id: TCGA-A1-A0SB-01 diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGsvaImporter.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGsvaImporter.java index 3220f0ee..bea888a4 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGsvaImporter.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalGsvaImporter.java @@ -28,14 +28,9 @@ import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportTabDelimData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; /** @@ -46,8 +41,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalGsvaImporter extends IntegrationTestBase { @Test public void testGsvaIsNotSupported() throws DaoException, IOException { diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMrnaExpressionImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMrnaExpressionImport.java index a824a900..9c4be884 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMrnaExpressionImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMrnaExpressionImport.java @@ -19,19 +19,15 @@ import java.io.*; import java.util.*; -import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; -import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneticAlteration; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -46,8 +42,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestIncrementalMrnaExpressionImport extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMutationsImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMutationsImport.java index 8c0071a0..2d0211c9 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMutationsImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalMutationsImport.java @@ -35,10 +35,8 @@ import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -52,8 +50,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalMutationsImport extends IntegrationTestBase { public static final String STUDY_ID = "study_tcga_pub"; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalPatientsImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalPatientsImport.java index 2e93a23c..e0af4934 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalPatientsImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalPatientsImport.java @@ -31,10 +31,8 @@ import org.mskcc.cbio.portal.model.ClinicalData; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.scripts.ImportClinicalData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -47,8 +45,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalPatientsImport extends IntegrationTestBase { public static final String STUDY_ID = "study_tcga_pub"; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalProteinLevelImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalProteinLevelImport.java index 0413392e..ae03359c 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalProteinLevelImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalProteinLevelImport.java @@ -28,10 +28,8 @@ import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -46,8 +44,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestIncrementalProteinLevelImport extends IntegrationTestBase { /** diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalSamplesImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalSamplesImport.java index aca011db..6632d523 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalSamplesImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalSamplesImport.java @@ -20,6 +20,8 @@ import java.io.*; import java.util.*; import java.util.stream.Collectors; + +import org.junit.After; import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; @@ -37,10 +39,8 @@ import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportClinicalData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -53,8 +53,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalSamplesImport extends IntegrationTestBase { public static final String STUDY_ID = "study_tcga_pub"; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalStructuralVariantsImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalStructuralVariantsImport.java index 0802d20a..10ef209c 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalStructuralVariantsImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalStructuralVariantsImport.java @@ -22,6 +22,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGenePanel; @@ -29,22 +30,18 @@ import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.dao.DaoStructuralVariant; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.GenePanel; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.model.StructuralVariant; import org.mskcc.cbio.portal.scripts.ImportProfileData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.mskcc.cbio.portal.dao.DaoMutation.getMutations; /** * Tests Incremental Import of Structural Variants Data. @@ -54,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalStructuralVariantsImport extends IntegrationTestBase { public static final String STUDY_ID = "study_tcga_pub"; @@ -87,7 +82,7 @@ public void testIncrementalUpload() throws DaoException { structuralVariant.setSite2RegionNumber(2); structuralVariant.setComments("This record has to be overwritten"); DaoStructuralVariant.addStructuralVariantToBulkLoader(structuralVariant); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); DaoSampleProfile.upsertSampleToProfileMapping(List.of( new DaoSampleProfile.SampleProfileTuple(svGeneticProfile.getGeneticProfileId(), svDataSample.getInternalId(), null))); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTabDelimDataTransaction.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTabDelimDataTransaction.java index 6afa2dd9..ce71fbde 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTabDelimDataTransaction.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTabDelimDataTransaction.java @@ -20,22 +20,19 @@ import java.io.*; import java.util.*; import org.junit.Before; +import org.junit.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.runner.RunWith; -import org.junit.Test; import org.mockito.junit.jupiter.MockitoExtension; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; import org.mskcc.cbio.portal.dao.DaoGeneticAlteration; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; -import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.ImportTabDelimData; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -57,10 +54,9 @@ public class TestIncrementalTabDelimDataTransaction extends IntegrationTestBase /** * Test transaction */ - @Test @ExtendWith(MockitoExtension.class) - //Mysql does not support nested transactions. That's why we disable the outer transaction. - @Transactional(propagation = Propagation.NOT_SUPPORTED) + @Test + //FIXME will pass when transaction substitution for ClickHouse is implemented public void testTransaction() throws Exception { GeneticProfile mrnaProfile = DaoGeneticProfile.getGeneticProfileByStableId("study_tcga_pub_mrna"); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTimelineImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTimelineImport.java index 00b04ca6..b822128b 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTimelineImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestIncrementalTimelineImport.java @@ -22,19 +22,17 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalEvent; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoPatient; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.ClinicalEvent; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.scripts.ImportTimelineData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -47,8 +45,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIncrementalTimelineImport extends IntegrationTestBase { public static final String STUDY_ID = "study_tcga_pub"; @@ -61,7 +57,7 @@ public void setUp() throws DaoException { @Test public void testTimelineDataReloading() throws DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); ClinicalEvent event = new ClinicalEvent(); event.setClinicalEventId(1L); Patient sbPatient = DaoPatient.getPatientByCancerStudyAndPatientId(cancerStudy.getInternalId(), "TCGA-A1-A0SB"); @@ -70,7 +66,7 @@ public void testTimelineDataReloading() throws DaoException { event.setEventType("SPECIMEN"); event.setEventData(Map.of("SPECIMEN_SITE", "specimen_site_to_erase")); DaoClinicalEvent.addClinicalEvent(event); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); File singleTcgaSampleFolder = new File("src/test/resources/incremental/clinical/"); File metaFile = new File(singleTcgaSampleFolder, "meta_timeline.txt"); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestUpdateCaseListsSampleIds.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestUpdateCaseListsSampleIds.java index 9545ce0f..28dbf39e 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestUpdateCaseListsSampleIds.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/incremental/TestUpdateCaseListsSampleIds.java @@ -26,10 +26,8 @@ import org.mskcc.cbio.portal.dao.DaoSampleList; import org.mskcc.cbio.portal.model.SampleList; import org.mskcc.cbio.portal.scripts.UpdateCaseListsSampleIds; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -42,8 +40,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestUpdateCaseListsSampleIds extends IntegrationTestBase { DaoSampleList daoSampleList = new DaoSampleList(); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/pancancer/TestPanCancerStudySummaryImport.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/pancancer/TestPanCancerStudySummaryImport.java index ea627431..9509e746 100755 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/pancancer/TestPanCancerStudySummaryImport.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/pancancer/TestPanCancerStudySummaryImport.java @@ -3,7 +3,6 @@ import java.util.*; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.runner.RunWith; import org.junit.Test; import org.mskcc.cbio.portal.dao.DaoCancerStudy; @@ -25,18 +24,13 @@ import org.mskcc.cbio.portal.scripts.ImportSampleList; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @RunWith(SpringJUnit4ClassRunner.class) -@Ignore @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestPanCancerStudySummaryImport extends IntegrationTestBase { @Before diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestDumpPortalInfoApproval.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestDumpPortalInfoApproval.java index b0cbaee5..d27f70c2 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestDumpPortalInfoApproval.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestDumpPortalInfoApproval.java @@ -68,7 +68,7 @@ private static void assertApprovedFilesMatch(Path receivedDir, Path approvedDir) private static void assertJsonMatch(String fileName, Path approved, Path received) throws JSONException, IOException { String message = "Mismatch for " + fileName + ".\nCopy " + received.toAbsolutePath() + " to " + approved.toAbsolutePath() + " after review."; - JSONAssert.assertEquals(message, Files.readString(approved), Files.readString(received), true); + JSONAssert.assertEquals(message, Files.readString(approved), Files.readString(received), false); } private static void assertNoUnexpectedFiles(Path receivedDir) throws IOException { diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportClinicalData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportClinicalData.java index 4f59425d..a49beaf7 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportClinicalData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportClinicalData.java @@ -34,6 +34,7 @@ import java.io.*; import java.util.*; + import org.junit.Before; import org.junit.Rule; import org.junit.rules.ExpectedException; @@ -48,10 +49,8 @@ import org.mskcc.cbio.portal.scripts.ImportClinicalData; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -66,8 +65,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportClinicalData extends IntegrationTestBase { //To use in test cases where we expect an exception: @@ -153,7 +150,7 @@ public void testImportPatientDataNewStudy_WithDuplError() throws Exception { importClinicalData.setFile(cancerStudy, clinicalFile, "PATIENT_ATTRIBUTES", false); exception.expect(RuntimeException.class); - exception.expectMessage("Duplicated patient"); + exception.expectMessage("Error: Duplicated patient TEST-BH-A1F0 and SUBTYPE attribute pair in the clinical file."); importClinicalData.importData(); ConsoleUtil.showWarnings(); } @@ -423,12 +420,13 @@ public void testImportClinicalData_PatientIdError() throws Exception { */ @Test public void testImportClinicalData_CorrectFileTwice1() throws Exception { - checkCorrectFileTwice("PATIENT"); + checkCorrectFileTwice("PATIENT", "Error: The following patient internal id and attribute entries already exist in the database:"); } + @Test public void testImportClinicalData_CorrectFileTwice2() throws Exception { - checkCorrectFileTwice("SAMPLE"); + checkCorrectFileTwice("SAMPLE", "Error: Sample TEST-A2-A04P is already in the database."); } - private void checkCorrectFileTwice(String type) throws Exception { + private void checkCorrectFileTwice(String type, String errorMessage) throws Exception { File clinicalFile = new File("src/test/resources/clinical_data_small_" + type + ".txt"); // initialize an ImportClinicalData instance without args to parse ImportClinicalData importClinicalData = new ImportClinicalData(null); @@ -436,11 +434,8 @@ private void checkCorrectFileTwice(String type) throws Exception { importClinicalData.setFile(cancerStudy, clinicalFile, type + "_ATTRIBUTES", false); importClinicalData.importData(); // loading twice should also give error - exception.expect(DaoException.class); - // it is not a specific "duplication" error message but a general DB error since the - // validation only gives specific error when in same file (maybe at some point we want to support clinical data - // in multiple PATIENT and SAMPLE files(?) ): - exception.expectMessage("DB Error"); + exception.expect(RuntimeException.class); + exception.expectMessage(errorMessage); importClinicalData.importData(); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCnaDiscreteLongData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCnaDiscreteLongData.java index 16e3e23f..2f90e102 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCnaDiscreteLongData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCnaDiscreteLongData.java @@ -38,6 +38,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoCnaEvent; import org.mskcc.cbio.portal.dao.DaoException; @@ -47,7 +48,6 @@ import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.dao.JdbcUtil; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CnaEvent; import org.mskcc.cbio.portal.model.CopyNumberStatus; import org.mskcc.cbio.portal.model.GeneticProfile; @@ -55,10 +55,8 @@ import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportCnaDiscreteLongData; import org.mskcc.cbio.portal.util.StableIdUtil; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Sets.newHashSet; @@ -70,8 +68,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestImportCnaDiscreteLongData extends IntegrationTestBase { int studyId; GeneticProfile geneticProfile; @@ -89,7 +85,7 @@ public void setUp() throws DaoException { @After public void cleanUp() throws DaoException { - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } /** @@ -556,10 +552,10 @@ private void assertSampleExistsInGeneticProfile(String sampleId) throws DaoExcep private String getGeneticProfileDatatype(long geneticProfileId) throws DaoException { return runSelectQuery( - "select DATATYPE " + + "select datatype " + "FROM genetic_profile " + - "WHERE GENETIC_PROFILE_ID=" + geneticProfileId + " ;", - (ResultSet rs) -> rs.getString("DATATYPE") + "WHERE genetic_profile_id=" + geneticProfileId + " ;", + (ResultSet rs) -> rs.getString("datatype") ).get(0); } @@ -573,18 +569,18 @@ class NamespaceAnnotationJson { private List getAnnotationJsonBy(long geneticProfileId) throws DaoException { return runSelectQuery( - "select cna_event.CNA_EVENT_ID, SAMPLE_ID, STABLE_ID, GENETIC_PROFILE_ID, ENTREZ_GENE_ID, ANNOTATION_JSON " + + "select cna_event.cna_event_id as cna_event_id, sample_id, stable_id, genetic_profile_id, entrez_gene_id, annotation_json " + "FROM sample_cna_event " + - "LEFT JOIN sample ON sample.INTERNAL_ID = sample_cna_event.SAMPLE_ID " + - "LEFT JOIN cna_event ON cna_event.CNA_EVENT_ID = sample_cna_event.CNA_EVENT_ID " + - "WHERE GENETIC_PROFILE_ID=" + geneticProfileId + " ;", + "LEFT JOIN sample ON sample.internal_id = sample_cna_event.sample_id " + + "LEFT JOIN cna_event ON cna_event.cna_event_id = sample_cna_event.cna_event_id " + + "WHERE genetic_profile_id=" + geneticProfileId + " ;", (ResultSet rs) -> { NamespaceAnnotationJson result = new NamespaceAnnotationJson(); - result.cnaEventId = rs.getInt("CNA_EVENT_ID"); - result.sampleId = rs.getInt("SAMPLE_ID"); - result.stableId = rs.getString("STABLE_ID"); - result.entrezGeneId = rs.getInt("ENTREZ_GENE_ID"); - result.annotationJson = rs.getString("ANNOTATION_JSON"); + result.cnaEventId = rs.getInt("cna_event_id"); + result.sampleId = rs.getInt("sample_id"); + result.stableId = rs.getString("stable_id"); + result.entrezGeneId = rs.getInt("entrez_gene_id"); + result.annotationJson = rs.getString("annotation_json"); return result; }); } @@ -593,13 +589,13 @@ private List getAllCnaPdAnnotations(List p List pkStrings = new ArrayList<>(); for (TestPdAnnotationPK pk : pks) { pkStrings.add(String.format( - "( ALTERATION_EVENT_ID=%d AND GENETIC_PROFILE_ID=%d AND SAMPLE_ID=%d )", + "( alteration_event_id=%d AND genetic_profile_id=%d AND sample_id=%d )", pk.alterationEventId, pk.geneticProfileId, pk.sampleId )); } - String q = "SELECT DRIVER_TIERS_FILTER_ANNOTATION, DRIVER_TIERS_FILTER, DRIVER_FILTER_ANNOTATION, DRIVER_FILTER, " - + "ALTERATION_EVENT_ID, GENETIC_PROFILE_ID, SAMPLE_ID " + String q = "SELECT driver_tiers_filter_annotation, driver_tiers_filter, driver_filter_annotation, driver_filter, " + + "alteration_event_id, genetic_profile_id, sample_id " + "FROM alteration_driver_annotation "; if (pks.size() > 0) { q += "WHERE " + String.join(" OR ", pkStrings); @@ -609,62 +605,62 @@ private List getAllCnaPdAnnotations(List p q, (ResultSet rs) -> { TestPdAnnotation line = new TestPdAnnotation(); - line.driverFilter = rs.getString("DRIVER_FILTER"); - line.driverTiersFilter = rs.getString("DRIVER_TIERS_FILTER"); - line.driverFilterAnnotation = rs.getString("DRIVER_FILTER_ANNOTATION"); - line.driverTiersFilterAnnotation = rs.getString("DRIVER_TIERS_FILTER_ANNOTATION"); + line.driverFilter = rs.getString("driver_filter"); + line.driverTiersFilter = rs.getString("driver_tiers_filter"); + line.driverFilterAnnotation = rs.getString("driver_filter_annotation"); + line.driverTiersFilterAnnotation = rs.getString("driver_tiers_filter_annotation"); line.pk = new TestPdAnnotationPK(); - line.pk.alterationEventId = rs.getInt("ALTERATION_EVENT_ID"); - line.pk.geneticProfileId = rs.getInt("GENETIC_PROFILE_ID"); - line.pk.sampleId = rs.getInt("SAMPLE_ID"); + line.pk.alterationEventId = rs.getInt("alteration_event_id"); + line.pk.geneticProfileId = rs.getInt("genetic_profile_id"); + line.pk.sampleId = rs.getInt("sample_id"); return line; }); } private List getAllGeneticAlterations() throws DaoException { - return runSelectQuery("SELECT ga.*, g.HUGO_GENE_SYMBOL FROM genetic_alteration as ga left join gene as g on ga.GENETIC_ENTITY_ID=g.GENETIC_ENTITY_ID", (ResultSet rs) -> { + return runSelectQuery("SELECT ga.*, g.hugo_gene_symbol FROM genetic_alteration as ga left join gene as g on ga.genetic_entity_id=g.genetic_entity_id", (ResultSet rs) -> { TestGeneticAlteration line = new TestGeneticAlteration(); - line.geneticProfileId = rs.getInt("GENETIC_PROFILE_ID"); - line.geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - line.values = rs.getString("VALUES"); - line.hugoGeneSymbol = rs.getString("HUGO_GENE_SYMBOL"); + line.geneticProfileId = rs.getInt("genetic_profile_id"); + line.geneticEntityId = rs.getInt("genetic_entity_id"); + line.values = rs.getString("values"); + line.hugoGeneSymbol = rs.getString("hugo_gene_symbol"); return line; }); } private TestGeneticAlteration getGeneticAlterationByEntrez(long entrezId) throws DaoException { - return runSelectQuery("SELECT ga.GENETIC_PROFILE_ID, ga.GENETIC_ENTITY_ID, ga.VALUES, g.HUGO_GENE_SYMBOL " + + return runSelectQuery("SELECT ga.genetic_profile_id, ga.genetic_entity_id, ga.values, g.hugo_gene_symbol " + "FROM genetic_alteration AS ga " + "RIGHT JOIN gene AS g " + - "ON g.GENETIC_ENTITY_ID = ga.GENETIC_ENTITY_ID " + - "WHERE g.ENTREZ_GENE_ID=" + entrezId, + "ON g.genetic_entity_id = ga.genetic_entity_id " + + "WHERE g.entrez_gene_id=" + entrezId, (ResultSet rs) -> { TestGeneticAlteration line = new TestGeneticAlteration(); - line.geneticProfileId = rs.getInt("GENETIC_PROFILE_ID"); - line.geneticEntityId = rs.getInt("GENETIC_ENTITY_ID"); - line.values = rs.getString("VALUES"); - line.hugoGeneSymbol = rs.getString("HUGO_GENE_SYMBOL"); + line.geneticProfileId = rs.getInt("genetic_profile_id"); + line.geneticEntityId = rs.getInt("genetic_entity_id"); + line.values = rs.getString("values"); + line.hugoGeneSymbol = rs.getString("hugo_gene_symbol"); return line; }).get(0); } private TestGeneticProfileSample getGeneticProfileSample(long profileId) throws DaoException { return runSelectQuery( - "SELECT * FROM genetic_profile_samples WHERE GENETIC_PROFILE_ID=" + profileId, + "SELECT * FROM genetic_profile_samples WHERE genetic_profile_id=" + profileId, (ResultSet rs) -> { TestGeneticProfileSample line = new TestGeneticProfileSample(); - line.geneticProfileId = rs.getInt("GENETIC_PROFILE_ID"); - line.orderedSampleList = rs.getString("ORDERED_SAMPLE_LIST"); + line.geneticProfileId = rs.getInt("genetic_profile_id"); + line.orderedSampleList = rs.getString("ordered_sample_list"); return line; }).get(0); } private String getSampleStableIdFromInternalId(Integer internalSampleId) throws DaoException { return runSelectQuery( - "select STABLE_ID from sample where INTERNAL_ID = " + internalSampleId, - (ResultSet rs) -> rs.getString("STABLE_ID") + "select stable_id from sample where internal_id = " + internalSampleId, + (ResultSet rs) -> rs.getString("stable_id") ).get(0); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCopyNumberSegmentData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCopyNumberSegmentData.java index e9150952..2a0228dd 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCopyNumberSegmentData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportCopyNumberSegmentData.java @@ -29,19 +29,17 @@ import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.ImportCopyNumberSegmentData; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; /** @@ -50,8 +48,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportCopyNumberSegmentData extends IntegrationTestBase { //To use in test cases where we expect an exception: @@ -103,6 +99,6 @@ private void addTestPatientAndSampleRecords(File file, CancerStudy cancerStudy) Integer pId = (p == null) ? DaoPatient.addPatient(new Patient(cancerStudy, sampleId)) : p.getInternalId(); DaoSample.addSample(new Sample(sampleId, pId, cancerStudy.getTypeOfCancerId())); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportExtendedMutationData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportExtendedMutationData.java index 3f91db81..9a62db0d 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportExtendedMutationData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportExtendedMutationData.java @@ -37,6 +37,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalAttributeMeta; import org.mskcc.cbio.portal.dao.DaoClinicalData; @@ -46,7 +47,6 @@ import org.mskcc.cbio.portal.dao.DaoMutation; import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ClinicalAttribute; @@ -61,10 +61,8 @@ import org.mskcc.cbio.portal.util.ProgressMonitor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static com.google.common.collect.Sets.newHashSet; import static org.junit.Assert.assertEquals; @@ -76,8 +74,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/integrationTestScript.xml", "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportExtendedMutationData extends IntegrationTestBase { @Autowired @@ -120,13 +116,13 @@ public void setUp() throws DaoException { @Test public void testImportExtendedMutationDataExtended() throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); // TBD: change this to use getResourceAsStream() File file = new File("src/test/resources/data_mutations_extended.txt"); ImportExtendedMutationData parser = new ImportExtendedMutationData(file, geneticProfileId, null); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ConsoleUtil.showMessages(); int sampleId = DaoSample.getSampleByCancerStudyAndSampleId(studyId, "TCGA-AA-3664-01").getInternalId(); @@ -149,7 +145,7 @@ public void testImportExtendedMutationDataOncotated() throws IOException, DaoExc File file = new File("src/test/resources/data_mutations_oncotated.txt"); ImportExtendedMutationData parser = new ImportExtendedMutationData(file, geneticProfileId, null); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ArrayList mutationList = DaoMutation.getAllMutations(geneticProfileId); @@ -172,13 +168,13 @@ public void testImportExtendedMutationDataOncotated() throws IOException, DaoExc @Test public void testImportExtendedMutationDataExtendedCustomFiltering() throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); File file = new File("src/test/resources/data_mutations_extended.txt"); Set customFiltering = new HashSet(Arrays.asList("Missense_Mutation", "Nonsense_Mutation")); ImportExtendedMutationData parser = new ImportExtendedMutationData(file, geneticProfileId, null, customFiltering, null); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ConsoleUtil.showMessages(); rejectMissenseAndNonsenseMutations(); @@ -192,13 +188,13 @@ public void testImportExtendedMutationDataExtendedCustomFiltering() throws IOExc @Test public void testImportExtendedMutationDataExtendedNoFiltering() throws IOException, DaoException { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); File file = new File("src/test/resources/data_mutations_extended.txt"); Set customFiltering = new HashSet(Arrays.asList("")); ImportExtendedMutationData parser = new ImportExtendedMutationData(file, geneticProfileId, null, customFiltering, null); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ConsoleUtil.showMessages(); acceptAllMutationTypes(); @@ -309,7 +305,7 @@ private void loadGenes() throws DaoException { daoGene.addGene(new CanonicalGene(2322L, "FLT3")); daoGene.addGene(new CanonicalGene(867L, "CBL")); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } /** @@ -367,11 +363,11 @@ private void acceptAllMutationTypes() throws DaoException { @Test public void testImportExtendedMutationDataExtendedWithSharedMutationEvent() throws IOException, DaoException { // import maf - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); File file = new File("src/test/resources/data_mutations_extended_duplicate_events.txt"); ImportExtendedMutationData parser = new ImportExtendedMutationData(file, testGeneticProfileId, null); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ConsoleUtil.showMessages(); // fetch mutations for test genetic profile int geneticProfileId = DaoGeneticProfile.getGeneticProfileByStableId("test_importer_id").getGeneticProfileId(); @@ -389,13 +385,13 @@ public void testImportExtendedMutationDataExtendedWithSharedMutationEvent() thro @Test public void testImportExtendedMutationDataExtendedWithoutNamespacesResultsInNull() throws IOException, DaoException { // import maf - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); File file = new File("src/test/resources/data_mutations_extended_without_namespaces.txt"); ImportExtendedMutationData parser = new ImportExtendedMutationData( file, testGeneticProfileId, null, null, newHashSet("foo-namespace", "bar-namespace") ); parser.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); ConsoleUtil.showMessages(); // fetch mutations for test genetic profile int geneticProfileId = DaoGeneticProfile.getGeneticProfileByStableId("test_importer_id").getGeneticProfileId(); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGeneData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGeneData.java index 0bb44b27..3664bc2b 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGeneData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGeneData.java @@ -39,10 +39,8 @@ import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.scripts.ImportGeneData; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -51,8 +49,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportGeneData extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayData.java index 1dca3d4e..8c419200 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayData.java @@ -42,10 +42,8 @@ import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.scripts.ImportGenericAssayEntity; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -56,8 +54,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportGenericAssayData extends IntegrationTestBase { @Test @@ -129,7 +125,7 @@ private int getNumRecordsForGenericAssay() { ResultSet rs = null; try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - stat = con.prepareStatement("SELECT COUNT(*) FROM genetic_entity WHERE ENTITY_TYPE = 'GENERIC_ASSAY'"); + stat = con.prepareStatement("SELECT COUNT(*) FROM genetic_entity WHERE entity_type = 'GENERIC_ASSAY'"); rs = stat.executeQuery(); if (rs.next()) { return rs.getInt(1); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayPatientLevelData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayPatientLevelData.java index b922203a..d8f35d03 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayPatientLevelData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenericAssayPatientLevelData.java @@ -37,6 +37,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; @@ -47,7 +48,6 @@ import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; @@ -56,10 +56,8 @@ import org.mskcc.cbio.portal.model.shared.GeneticEntity; import org.mskcc.cbio.portal.scripts.ImportGenericAssayPatientLevelData; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -69,8 +67,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportGenericAssayPatientLevelData extends IntegrationTestBase { private int studyId; @@ -123,7 +119,7 @@ public void setUp() throws DaoException { @Test public void testImportGenericAssayPatientLevelDataBulkLoadOff() throws Exception { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); runImportGenericAssayPatientLevelData(); } @@ -133,7 +129,7 @@ public void testImportGenericAssayPatientLevelDataBulkLoadOff() throws Exception */ @Test public void testImportGenericAssayPatientLevelDataBulkLoadOn() throws Exception { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); runImportGenericAssayPatientLevelData(); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetData.java index 108b172f..bfc4e913 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetData.java @@ -34,17 +34,13 @@ import org.mskcc.cbio.portal.model.Geneset; import org.mskcc.cbio.portal.scripts.ImportGenesetData; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportGenesetData extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetHierarchyData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetHierarchyData.java index 2fc42645..1309393f 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetHierarchyData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportGenesetHierarchyData.java @@ -46,10 +46,8 @@ import org.mskcc.cbio.portal.scripts.ImportGenesetData; import org.mskcc.cbio.portal.scripts.ImportGenesetHierarchy; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -59,8 +57,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportGenesetHierarchyData extends IntegrationTestBase { @Test @@ -110,7 +106,7 @@ public static GenesetHierarchy getGenesetHierarchyFromNodeId(int nodeId) throws connection = JdbcUtil.getDbConnection(DaoGenesetHierarchyNode.class); // Prepare SQL statement - preparedStatement = connection.prepareStatement("SELECT * FROM geneset_hierarchy_node WHERE NODE_ID = ?"); + preparedStatement = connection.prepareStatement("SELECT * FROM geneset_hierarchy_node WHERE node_id = ?"); preparedStatement.setInt(1, nodeId); // Execute statement @@ -119,9 +115,9 @@ public static GenesetHierarchy getGenesetHierarchyFromNodeId(int nodeId) throws // Extract genesetHierarchy values if (resultSet.next()) { GenesetHierarchy genesetHierarchy = new GenesetHierarchy(); - genesetHierarchy.setNodeId(resultSet.getInt("NODE_ID")); - genesetHierarchy.setNodeName(resultSet.getString("NODE_NAME")); - genesetHierarchy.setParentId(resultSet.getInt("PARENT_ID")); + genesetHierarchy.setNodeId(resultSet.getInt("node_id")); + genesetHierarchy.setNodeName(resultSet.getString("node_name")); + genesetHierarchy.setParentId(resultSet.getInt("parent_id")); return genesetHierarchy; } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportProfileData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportProfileData.java index 104bcb0f..4dbfa5a3 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportProfileData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportProfileData.java @@ -30,6 +30,7 @@ import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalAttributeMeta; import org.mskcc.cbio.portal.dao.DaoClinicalData; @@ -42,7 +43,6 @@ import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; import org.mskcc.cbio.portal.dao.DaoStructuralVariant; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ClinicalAttribute; @@ -57,10 +57,8 @@ import org.mskcc.cbio.portal.scripts.ImportProfileData; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -72,8 +70,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportProfileData extends IntegrationTestBase { int studyId; @@ -450,7 +446,7 @@ public void testImportCNAFile() throws Exception { DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance(); daoGene.addGene(new CanonicalGene(999999672, "TESTBRCA1")); daoGene.addGene(new CanonicalGene(999999675, "TESTBRCA2")); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); String[] args = { "--data","src/test/resources/data_CNA_sample.txt", "--meta","src/test/resources/meta_CNA.txt" , @@ -558,6 +554,6 @@ private static void loadGenes() throws DaoException { daoGene.addGene(new CanonicalGene(1952L, "CELSR2")); daoGene.addGene(new CanonicalGene(2322L, "FLT3")); daoGene.addGene(new CanonicalGene(867L, "CBL")); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportReferenceGenome.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportReferenceGenome.java index 1f8d6f00..f09ae13e 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportReferenceGenome.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportReferenceGenome.java @@ -7,10 +7,8 @@ import org.mskcc.cbio.portal.model.ReferenceGenome; import org.mskcc.cbio.portal.scripts.ImportReferenceGenome; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; @@ -19,8 +17,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportReferenceGenome extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportStructuralVariantData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportStructuralVariantData.java index c1ea461d..434f04ee 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportStructuralVariantData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportStructuralVariantData.java @@ -34,23 +34,20 @@ import junitparams.Parameters; import org.junit.Before; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneticProfile; import org.mskcc.cbio.portal.dao.DaoStructuralVariant; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.StructuralVariant; import org.mskcc.cbio.portal.scripts.ImportStructuralVariantData; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.rules.SpringClassRule; import org.springframework.test.context.junit4.rules.SpringMethodRule; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static com.google.common.collect.Sets.newHashSet; import static org.junit.Assert.assertEquals; @@ -59,11 +56,8 @@ * Test class to test functionality of ImportStructralVariantData */ @RunWith(JUnitParamsRunner.class) -@Ignore @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportStructuralVariantData extends IntegrationTestBase { int studyId; int geneticProfileId; @@ -90,7 +84,7 @@ public void testImportStructuralVariantData() throws DaoException, IOException { File file = new File("src/test/resources/data_structural_variants.txt"); ImportStructuralVariantData importer = new ImportStructuralVariantData(file, geneticProfileId, null, noNamespaces, false); importer.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); List structuralVariants = DaoStructuralVariant.getAllStructuralVariants(); assertEquals("KIAA1549-BRAF.K16B10.COSF509_2", structuralVariants.get(0).getSite2Description()); @@ -132,7 +126,7 @@ public void testImportStructuralVariantDataImportsCustomNamespacesFromTwoSamples Set namespacesToImport = newHashSet("StructVarNamespace", "StructVarNamespace2"); ImportStructuralVariantData importer = new ImportStructuralVariantData(file, geneticProfileId, null, namespacesToImport, false); importer.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); List all = DaoStructuralVariant .getAllStructuralVariants(); @@ -158,7 +152,7 @@ public void testImportStructuralVariantDataIgnoresUnspecifiedNamespaces() throws Set namespacesToImport = newHashSet("StructVarNamespace", "StructVarNamespace2"); ImportStructuralVariantData importer = new ImportStructuralVariantData(file, geneticProfileId, null, namespacesToImport, false); importer.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); List all = DaoStructuralVariant .getAllStructuralVariants(); @@ -181,7 +175,7 @@ public void testImportStructuralVariantDataWithNoNamespaceData() throws DaoExcep Set namespacesToImport = newHashSet("StructVarNamespace", "StructVarNamespace2"); ImportStructuralVariantData importer = new ImportStructuralVariantData(file, geneticProfileId, null, namespacesToImport, false); importer.importData(); - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); List all = DaoStructuralVariant .getAllStructuralVariants(); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportTabDelimData.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportTabDelimData.java index 06a2f24e..27d640a2 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportTabDelimData.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportTabDelimData.java @@ -37,6 +37,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoException; import org.mskcc.cbio.portal.dao.DaoGeneOptimized; @@ -45,7 +46,6 @@ import org.mskcc.cbio.portal.dao.DaoPatient; import org.mskcc.cbio.portal.dao.DaoSample; import org.mskcc.cbio.portal.dao.DaoSampleProfile; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.CopyNumberStatus; @@ -56,10 +56,8 @@ import org.mskcc.cbio.portal.scripts.ImportTabDelimData; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -69,8 +67,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportTabDelimData extends IntegrationTestBase { private final Set NON_CASE_ID_COLS = new HashSet<>(Arrays.asList( @@ -125,7 +121,7 @@ public void setUp() throws DaoException { @Test public void testImportCnaDataBulkLoadOff() throws Exception { - MySQLbulkLoader.bulkLoadOff(); + ClickHouseBulkLoader.bulkLoadOff(); runImportCnaData(); } @@ -135,11 +131,11 @@ public void testImportCnaDataBulkLoadOff() throws Exception { */ @Test public void testImportCnaDataBulkLoadOn() throws Exception { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); runImportCnaData(); } - private void runImportCnaData() throws DaoException, IOException{ + private void runImportCnaData() throws Exception { DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance(); DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance(); @@ -197,8 +193,8 @@ private void runImportCnaData() throws DaoException, IOException{ */ @Test public void testImportCnaData2BulkLoadOff() throws Exception { - // test with both values of MySQLbulkLoader.isBulkLoad() - MySQLbulkLoader.bulkLoadOff(); + // test with both values of ClickHouseBulkLoader.isBulkLoad() + ClickHouseBulkLoader.bulkLoadOff(); runImportCnaData2(); } @@ -209,12 +205,12 @@ public void testImportCnaData2BulkLoadOff() throws Exception { */ @Test public void testImportCnaData2BulkLoadOn() throws Exception { - // test with both values of MySQLbulkLoader.isBulkLoad() - MySQLbulkLoader.bulkLoadOn(); + // test with both values of ClickHouseBulkLoader.isBulkLoad() + ClickHouseBulkLoader.bulkLoadOn(); runImportCnaData2(); } - private void runImportCnaData2() throws DaoException, IOException{ + private void runImportCnaData2() throws Exception { DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance(); @@ -259,8 +255,8 @@ private void runImportCnaData2() throws DaoException, IOException{ */ @Test public void testImportmRnaData1BulkLoadOff() throws Exception { - // test with both values of MySQLbulkLoader.isBulkLoad() - MySQLbulkLoader.bulkLoadOff(); + // test with both values of ClickHouseBulkLoader.isBulkLoad() + ClickHouseBulkLoader.bulkLoadOff(); runImportRnaData1(); } @@ -270,12 +266,12 @@ public void testImportmRnaData1BulkLoadOff() throws Exception { */ @Test public void testImportmRnaData1BulkLoadOn() throws Exception { - // test with both values of MySQLbulkLoader.isBulkLoad() - MySQLbulkLoader.bulkLoadOn(); + // test with both values of ClickHouseBulkLoader.isBulkLoad() + ClickHouseBulkLoader.bulkLoadOn(); runImportRnaData1(); } - private void runImportRnaData1() throws DaoException, IOException{ + private void runImportRnaData1() throws Exception { DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance(); DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance(); @@ -325,7 +321,7 @@ private void runImportRnaData1() throws DaoException, IOException{ */ @Test public void testImportmRnaData2() throws Exception { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance(); @@ -411,7 +407,7 @@ public void testImportmRnaData2() throws Exception { */ @Test public void testImportRppaData() throws Exception { - MySQLbulkLoader.bulkLoadOn(); + ClickHouseBulkLoader.bulkLoadOn(); DaoGeneOptimized daoGene = DaoGeneOptimized.getInstance(); DaoGeneticAlteration dao = DaoGeneticAlteration.getInstance(); @@ -506,6 +502,6 @@ private void addTestPatientAndSampleRecords(File file) throws FileNotFoundExcept Integer pId = (p == null) ? DaoPatient.addPatient(new Patient(study, sampleId)) : p.getInternalId(); DaoSample.addSample(new Sample(sampleId, pId, study.getTypeOfCancerId())); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportUsers.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportUsers.java index a061f1d9..c45cbdeb 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportUsers.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestImportUsers.java @@ -39,10 +39,8 @@ import org.mskcc.cbio.portal.model.User; import org.mskcc.cbio.portal.model.UserAuthorities; import org.mskcc.cbio.portal.scripts.ImportUsers; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -53,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestImportUsers extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestIntegrationTest.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestIntegrationTest.java index e718faa0..54592635 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestIntegrationTest.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestIntegrationTest.java @@ -34,6 +34,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.junit.Test; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.dao.DaoClinicalAttributeMeta; import org.mskcc.cbio.portal.dao.DaoClinicalData; @@ -49,7 +50,6 @@ import org.mskcc.cbio.portal.dao.DaoSampleList; import org.mskcc.cbio.portal.dao.DaoStructuralVariant; import org.mskcc.cbio.portal.dao.DaoTypeOfCancer; -import org.mskcc.cbio.portal.dao.MySQLbulkLoader; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ClinicalAttribute; @@ -69,10 +69,8 @@ import org.mskcc.cbio.portal.util.TransactionalScripts; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -90,8 +88,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/integrationTestScript.xml", "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestIntegrationTest extends IntegrationTestBase { @Autowired @@ -449,7 +445,7 @@ private void loadGenes() throws DaoException, JsonParseException, JsonMappingExc daoGene.addGene(gene); } - MySQLbulkLoader.flushAll(); + ClickHouseBulkLoader.flushAll(); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestMutationFilter.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestMutationFilter.java index 82cb55bb..b886f2af 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestMutationFilter.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestMutationFilter.java @@ -40,10 +40,8 @@ import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.model.ExtendedMutation; import org.mskcc.cbio.portal.scripts.MutationFilter; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -53,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestMutationFilter extends IntegrationTestBase { @Before diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestNormalizeExpressionLevels.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestNormalizeExpressionLevels.java index 6009e99a..697bb4cf 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestNormalizeExpressionLevels.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestNormalizeExpressionLevels.java @@ -41,10 +41,8 @@ import org.mskcc.cbio.portal.dao.DaoGeneOptimized; import org.mskcc.cbio.portal.model.CanonicalGene; import org.mskcc.cbio.portal.scripts.NormalizeExpressionLevels; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -52,8 +50,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestNormalizeExpressionLevels extends IntegrationTestBase { private String validationFile; diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemovePatients.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemovePatients.java index 2958179f..9d8cefef 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemovePatients.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemovePatients.java @@ -33,10 +33,8 @@ import org.mskcc.cbio.portal.model.Patient; import org.mskcc.cbio.portal.model.Sample; import org.mskcc.cbio.portal.scripts.RemovePatients; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; @@ -53,8 +51,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestRemovePatients extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemoveSamples.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemoveSamples.java index 45fac9d8..da8e16a0 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemoveSamples.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/scripts/TestRemoveSamples.java @@ -30,10 +30,8 @@ import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.scripts.RemoveSamples; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; @@ -50,8 +48,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestRemoveSamples extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestCancerStudyReader.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestCancerStudyReader.java index 567bb8e2..b0f66044 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestCancerStudyReader.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestCancerStudyReader.java @@ -38,10 +38,8 @@ import org.mskcc.cbio.portal.dao.DaoCancerStudy; import org.mskcc.cbio.portal.model.CancerStudy; import org.mskcc.cbio.portal.util.CancerStudyReader; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -51,8 +49,6 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestCancerStudyReader extends IntegrationTestBase { @Test diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGenePanelUtil.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGenePanelUtil.java index adee6d48..17ac8196 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGenePanelUtil.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGenePanelUtil.java @@ -4,10 +4,8 @@ import org.junit.runner.RunWith; import org.junit.Test; import org.mskcc.cbio.portal.model.CanonicalGene; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -19,8 +17,6 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = {"classpath:/applicationContext-dao.xml"}) -@Rollback -@Transactional public class TestGenePanelUtil extends IntegrationTestBase { CanonicalGene otor = makeGene(13321, 56914, "OTOR", makeAliases("FDP", "MIAL1")); diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGeneticProfileReader.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGeneticProfileReader.java index 151d716b..73224282 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGeneticProfileReader.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestGeneticProfileReader.java @@ -43,20 +43,17 @@ import org.mskcc.cbio.portal.model.shared.GeneticAlterationType; import org.mskcc.cbio.portal.model.GeneticProfile; import org.mskcc.cbio.portal.util.GeneticProfileReader; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * JUnit test for GeneticProfileReader class. */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestGeneticProfileReader extends IntegrationTestBase { @Test @@ -71,12 +68,11 @@ public void testGeneticProfileReader() throws Exception { assertEquals("Blah Blah.", geneticProfile.getProfileDescription()); CancerStudy cancerStudy = DaoCancerStudy.getCancerStudyByStableId("study_tcga_pub"); - ArrayList list = DaoGeneticProfile.getAllGeneticProfiles - (cancerStudy.getInternalId()); - geneticProfile = list.get(0); - assertEquals(cancerStudy.getInternalId(), geneticProfile.getCancerStudyId()); - assertEquals("Putative copy-number alterations from GISTIC", geneticProfile.getProfileName()); + + List profileNames = DaoGeneticProfile.getAllGeneticProfiles + (cancerStudy.getInternalId()).stream().map(GeneticProfile::getProfileName).toList(); + assertTrue(profileNames.contains("Putative copy-number alterations from GISTIC")); assertEquals(GeneticAlterationType.COPY_NUMBER_ALTERATION, geneticProfile.getGeneticAlterationType()); } diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestMutSigReader.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestMutSigReader.java index 8ef8f046..9cff7ec8 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestMutSigReader.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestMutSigReader.java @@ -44,18 +44,14 @@ import org.mskcc.cbio.portal.model.MutSig; import org.mskcc.cbio.portal.util.MutSigReader; import org.mskcc.cbio.portal.util.ProgressMonitor; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestMutSigReader extends IntegrationTestBase { // TBD: change these to use getResourceAsStream() diff --git a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestTransactionalScripts.java b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestTransactionalScripts.java index 60f78455..c3299d3f 100644 --- a/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestTransactionalScripts.java +++ b/src/test/java/org/mskcc/cbio/portal/integrationTest/util/TestTransactionalScripts.java @@ -7,16 +7,12 @@ import org.mskcc.cbio.portal.util.TransactionalScripts; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.transaction.annotation.Transactional; import org.mskcc.cbio.portal.integrationTest.IntegrationTestBase; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = { "classpath:/demoScript.xml", "classpath:/applicationContext-dao.xml" }) -@Rollback -@Transactional public class TestTransactionalScripts extends IntegrationTestBase { @Autowired diff --git a/src/test/resources/applicationContext-dao.xml b/src/test/resources/applicationContext-dao.xml index 21044efa..ba71797b 100644 --- a/src/test/resources/applicationContext-dao.xml +++ b/src/test/resources/applicationContext-dao.xml @@ -44,8 +44,8 @@ - - + + diff --git a/src/test/resources/clickhouse_cgds.sql b/src/test/resources/clickhouse_cgds.sql new file mode 100644 index 00000000..5d451d5c --- /dev/null +++ b/src/test/resources/clickhouse_cgds.sql @@ -0,0 +1,762 @@ +/* +DROP TABLE IF EXISTS allele_specific_copy_number; +DROP TABLE IF EXISTS alteration_driver_annotation; +DROP TABLE IF EXISTS authorities; +DROP TABLE IF EXISTS cancer_study; +DROP TABLE IF EXISTS cancer_study_tags; +DROP TABLE IF EXISTS clinical_attribute_meta; +DROP TABLE IF EXISTS clinical_data_derived; +DROP TABLE IF EXISTS clinical_event; +DROP TABLE IF EXISTS clinical_event_data; +DROP TABLE IF EXISTS clinical_event_derived; +DROP TABLE IF EXISTS clinical_patient; +DROP TABLE IF EXISTS clinical_sample; +DROP TABLE IF EXISTS cna_event; +DROP TABLE IF EXISTS copy_number_seg; +DROP TABLE IF EXISTS copy_number_seg_file; +DROP TABLE IF EXISTS data_access_tokens; +DROP TABLE IF EXISTS gene; +DROP TABLE IF EXISTS gene_alias; +DROP TABLE IF EXISTS gene_panel; +DROP TABLE IF EXISTS gene_panel_list; +DROP TABLE IF EXISTS gene_panel_to_gene_derived; +DROP TABLE IF EXISTS generic_assay_data_derived; +DROP TABLE IF EXISTS generic_entity_properties; +DROP TABLE IF EXISTS geneset; +DROP TABLE IF EXISTS geneset_gene; +DROP TABLE IF EXISTS geneset_hierarchy_leaf; +DROP TABLE IF EXISTS geneset_hierarchy_node; +DROP TABLE IF EXISTS genetic_alteration; +DROP TABLE IF EXISTS genetic_alteration_derived; +DROP TABLE IF EXISTS genetic_entity; +DROP TABLE IF EXISTS genetic_profile; +DROP TABLE IF EXISTS genetic_profile_link; +DROP TABLE IF EXISTS genetic_profile_samples; +DROP TABLE IF EXISTS genomic_event_derived; +DROP TABLE IF EXISTS gistic; +DROP TABLE IF EXISTS gistic_to_gene; +DROP TABLE IF EXISTS info; +DROP TABLE IF EXISTS mut_sig; +DROP TABLE IF EXISTS mutation; +DROP TABLE IF EXISTS mutation_count_by_keyword; +DROP TABLE IF EXISTS mutation_event; +DROP TABLE IF EXISTS patient; +DROP TABLE IF EXISTS reference_genome; +DROP TABLE IF EXISTS reference_genome_gene; +DROP TABLE IF EXISTS resource_definition; +DROP TABLE IF EXISTS resource_node; +DROP TABLE IF EXISTS sample; +DROP TABLE IF EXISTS sample_cna_event; +DROP TABLE IF EXISTS sample_derived; +DROP TABLE IF EXISTS sample_list; +DROP TABLE IF EXISTS sample_list_list; +DROP TABLE IF EXISTS sample_profile; +DROP TABLE IF EXISTS sample_to_gene_panel_derived; +DROP TABLE IF EXISTS structural_variant; +DROP TABLE IF EXISTS type_of_cancer; +DROP TABLE IF EXISTS users; + */ + +CREATE TABLE allele_specific_copy_number +( + `mutation_event_id` Nullable(Int64), + `genetic_profile_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `ascn_integer_copy_number` Nullable(Int64), + `ascn_method` Nullable(String), + `ccf_expected_copies_upper` Nullable(Float64), + `ccf_expected_copies` Nullable(Float64), + `clonal` Nullable(String), + `minor_copy_number` Nullable(Int64), + `expected_alt_copies` Nullable(Int64), + `total_copy_number` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE alteration_driver_annotation +( + `alteration_event_id` Nullable(Int64), + `genetic_profile_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `driver_filter` Nullable(String), + `driver_filter_annotation` Nullable(String), + `driver_tiers_filter` Nullable(String), + `driver_tiers_filter_annotation` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE authorities +( + `email` Nullable(String), + `authority` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE cancer_study +( + `cancer_study_id` Nullable(Int64), + `cancer_study_identifier` Nullable(String), + `type_of_cancer_id` Nullable(String), + `name` Nullable(String), + `description` Nullable(String), + `public` Nullable(Int32), + `pmid` Nullable(String), + `citation` Nullable(String), + `groups` Nullable(String), + `status` Nullable(Int64), + `import_date` Nullable(DateTime64(6)), + `reference_genome_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE cancer_study_tags +( + `cancer_study_id` Nullable(Int64), + `tags` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE clinical_attribute_meta +( + `attr_id` Nullable(String), + `display_name` Nullable(String), + `description` Nullable(String), + `datatype` Nullable(String), + `patient_attribute` Nullable(Int32), + `priority` Nullable(String), + `cancer_study_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE clinical_data_derived +( + `internal_id` Int32, + `sample_unique_id` String, + `patient_unique_id` String, + `attribute_name` LowCardinality(String), + `attribute_value` String, + `cancer_study_identifier` LowCardinality(String), + `type` LowCardinality(String) +) + ENGINE = MergeTree +ORDER BY (type, attribute_name, sample_unique_id); + +CREATE TABLE clinical_event +( + `clinical_event_id` Nullable(Int64), + `patient_id` Nullable(Int64), + `start_date` Nullable(Int64), + `stop_date` Nullable(Int64), + `event_type` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE clinical_event_data +( + `clinical_event_id` Nullable(Int64), + `key` Nullable(String), + `value` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE clinical_event_derived +( + `patient_unique_id` String, + `key` String, + `value` String, + `start_date` Int32, + `stop_date` Int32 DEFAULT 0, + `event_type` LowCardinality(String), + `cancer_study_identifier` LowCardinality(String) +) + ENGINE = MergeTree +ORDER BY (event_type, patient_unique_id, cancer_study_identifier); + +CREATE TABLE clinical_patient +( + `internal_id` Int64, + `attr_id` String, + `attr_value` String +) + ENGINE = ReplacingMergeTree +ORDER BY tuple(`internal_id`, `attr_id`); + +CREATE TABLE clinical_sample +( + `internal_id` Int64, + `attr_id` String, + `attr_value` String +) + ENGINE = ReplacingMergeTree +ORDER BY tuple(`internal_id`, `attr_id`); + +CREATE TABLE cna_event +( + `cna_event_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64), + `alteration` Nullable(Int32) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE copy_number_seg +( + `seg_id` Nullable(Int64), + `cancer_study_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `chr` Nullable(String), + `start` Nullable(Int64), + `end` Nullable(Int64), + `num_probes` Nullable(Int64), + `segment_mean` Nullable(Float64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE copy_number_seg_file +( + `seg_file_id` Nullable(Int64), + `cancer_study_id` Nullable(Int64), + `reference_genome_id` Nullable(String), + `description` Nullable(String), + `filename` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE data_access_tokens +( + `token` Nullable(String), + `username` Nullable(String), + `expiration` Nullable(DateTime64(6)), + `creation` Nullable(DateTime64(6)) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gene +( + `entrez_gene_id` Nullable(Int64), + `hugo_gene_symbol` Nullable(String), + `genetic_entity_id` Nullable(Int64), + `type` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gene_alias +( + `entrez_gene_id` Nullable(Int64), + `gene_alias` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gene_panel +( + `internal_id` Nullable(Int64), + `stable_id` Nullable(String), + `description` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gene_panel_list +( + `internal_id` Nullable(Int64), + `gene_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gene_panel_to_gene_derived +( + `gene_panel_id` LowCardinality(String), + `gene` String +) + ENGINE = MergeTree +ORDER BY gene_panel_id; + +CREATE TABLE generic_assay_data_derived +( + `sample_unique_id` String, + `patient_unique_id` String, + `genetic_entity_id` String, + `value` String, + `generic_assay_type` String, + `profile_stable_id` String, + `entity_stable_id` String, + `datatype` String, + `patient_level` Decimal(10, 0), + `profile_type` String +) + ENGINE = MergeTree +ORDER BY (profile_type, entity_stable_id, patient_unique_id, sample_unique_id); + +CREATE TABLE generic_entity_properties +( + `id` Nullable(Int64), + `genetic_entity_id` Nullable(Int64), + `name` Nullable(String), + `value` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE geneset +( + `id` Nullable(Int64), + `genetic_entity_id` Nullable(Int64), + `external_id` Nullable(String), + `name` Nullable(String), + `description` Nullable(String), + `ref_link` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE geneset_gene +( + `geneset_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE geneset_hierarchy_leaf +( + `node_id` Nullable(Int64), + `geneset_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE geneset_hierarchy_node +( + `node_id` Nullable(Int64), + `node_name` Nullable(String), + `parent_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE genetic_alteration +( + `genetic_profile_id` Int64, + `genetic_entity_id` Int64, + `values` Nullable(String) +) + ENGINE = ReplacingMergeTree +ORDER BY tuple(`genetic_profile_id`, `genetic_entity_id`); + +CREATE TABLE genetic_alteration_derived +( + `sample_unique_id` String, + `cancer_study_identifier` LowCardinality(String), + `hugo_gene_symbol` String, + `profile_type` LowCardinality(String), + `alteration_value` Nullable(String) +) + ENGINE = MergeTree +ORDER BY (cancer_study_identifier, hugo_gene_symbol, profile_type, sample_unique_id); + +CREATE TABLE genetic_entity +( + `id` Nullable(Int64), + `entity_type` Nullable(String), + `stable_id` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE genetic_profile +( + `genetic_profile_id` Nullable(Int64), + `stable_id` Nullable(String), + `cancer_study_id` Nullable(Int64), + `genetic_alteration_type` Nullable(String), + `generic_assay_type` Nullable(String), + `datatype` Nullable(String), + `name` Nullable(String), + `description` Nullable(String), + `show_profile_in_analysis_tab` Nullable(Int32), + `pivot_threshold` Nullable(Float64), + `sort_order` Nullable(String), + `patient_level` Nullable(Int32) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE genetic_profile_link +( + `referring_genetic_profile_id` Nullable(Int64), + `referred_genetic_profile_id` Nullable(Int64), + `reference_type` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE genetic_profile_samples +( + `genetic_profile_id` Int64, + `ordered_sample_list` Nullable(String) +) + ENGINE = ReplacingMergeTree +ORDER BY tuple(`genetic_profile_id`); + +CREATE TABLE genomic_event_derived +( + `sample_unique_id` String, + `hugo_gene_symbol` String, + `entrez_gene_id` Int32, + `gene_panel_stable_id` LowCardinality(String), + `cancer_study_identifier` LowCardinality(String), + `genetic_profile_stable_id` LowCardinality(String), + `variant_type` LowCardinality(String), + `mutation_variant` String, + `mutation_type` LowCardinality(String), + `mutation_status` LowCardinality(String), + `driver_filter` LowCardinality(String), + `driver_tiers_filter` LowCardinality(String), + `cna_alteration` Nullable(Int8), + `cna_cytoband` String, + `sv_event_info` String, + `patient_unique_id` String, + `off_panel` Bool DEFAULT false +) + ENGINE = MergeTree +ORDER BY (variant_type, entrez_gene_id, hugo_gene_symbol, genetic_profile_stable_id, sample_unique_id); + +CREATE TABLE gistic +( + `gistic_roi_id` Nullable(Int64), + `cancer_study_id` Nullable(Int64), + `chromosome` Nullable(Int64), + `cytoband` Nullable(String), + `wide_peak_start` Nullable(Int64), + `wide_peak_end` Nullable(Int64), + `q_value` Nullable(Float64), + `amp` Nullable(Int32) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE gistic_to_gene +( + `gistic_roi_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE info +( + `db_schema_version` Nullable(String), + `geneset_version` Nullable(String), + `derived_table_schema_version` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE mut_sig +( + `cancer_study_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64), + `rank` Nullable(Int64), + `NumBasesCovered` Nullable(Int64), + `NumMutations` Nullable(Int64), + `p_value` Nullable(Float64), + `q_value` Nullable(Float64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE mutation +( + `mutation_event_id` Nullable(Int64), + `genetic_profile_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64), + `center` Nullable(String), + `sequencer` Nullable(String), + `mutation_status` Nullable(String), + `validation_status` Nullable(String), + `tumor_seq_allele1` Nullable(String), + `tumor_seq_allele2` Nullable(String), + `matched_norm_sample_barcode` Nullable(String), + `match_norm_seq_allele1` Nullable(String), + `match_norm_seq_allele2` Nullable(String), + `tumor_validation_allele1` Nullable(String), + `tumor_validation_allele2` Nullable(String), + `match_norm_validation_allele1` Nullable(String), + `match_norm_validation_allele2` Nullable(String), + `verification_status` Nullable(String), + `sequencing_phase` Nullable(String), + `sequence_source` Nullable(String), + `validation_method` Nullable(String), + `score` Nullable(String), + `bam_file` Nullable(String), + `tumor_alt_count` Nullable(Int64), + `tumor_ref_count` Nullable(Int64), + `normal_alt_count` Nullable(Int64), + `normal_ref_count` Nullable(Int64), + `amino_acid_change` Nullable(String), + `annotation_json` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE mutation_count_by_keyword +( + `genetic_profile_id` Nullable(Int64), + `keyword` Nullable(String), + `entrez_gene_id` Nullable(Int64), + `keyword_count` Nullable(Int64), + `gene_count` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE mutation_event +( + `mutation_event_id` Nullable(Int64), + `entrez_gene_id` Nullable(Int64), + `chr` Nullable(String), + `start_position` Nullable(Int64), + `end_position` Nullable(Int64), + `reference_allele` Nullable(String), + `tumor_seq_allele` Nullable(String), + `protein_change` Nullable(String), + `mutation_type` Nullable(String), + `ncbi_build` Nullable(String), + `strand` Nullable(String), + `variant_type` Nullable(String), + `db_snp_rs` Nullable(String), + `db_snp_val_status` Nullable(String), + `refseq_mrna_id` Nullable(String), + `codon_change` Nullable(String), + `uniprot_accession` Nullable(String), + `protein_pos_start` Nullable(Int64), + `protein_pos_end` Nullable(Int64), + `canonical_transcript` Nullable(Int32), + `keyword` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE patient +( + `internal_id` Nullable(Int64), + `stable_id` Nullable(String), + `cancer_study_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE reference_genome +( + `reference_genome_id` Nullable(Int64), + `species` Nullable(String), + `name` Nullable(String), + `build_name` Nullable(String), + `genome_size` Nullable(Int64), + `url` Nullable(String), + `release_date` Nullable(DateTime64(6)) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE reference_genome_gene +( + `entrez_gene_id` Nullable(Int64), + `reference_genome_id` Nullable(Int64), + `chr` Nullable(String), + `cytoband` Nullable(String), + `start` Nullable(Int64), + `end` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE resource_definition +( + `resource_id` Nullable(String), + `display_name` Nullable(String), + `description` Nullable(String), + `resource_type` Nullable(String), + `open_by_default` Nullable(Int32), + `priority` Nullable(Int64), + `cancer_study_id` Nullable(Int64), + `custom_metadata` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE resource_node +( + `id` Nullable(Int64), + `resource_id` Nullable(String), + `cancer_study_id` Nullable(Int64), + `entity_type` Nullable(String), + `entity_internal_id` Nullable(Int64), + `display_name` Nullable(String), + `url` Nullable(String), + `type` Nullable(String), + `metadata` Nullable(String), + `group_path` Nullable(String), + `priority` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE sample +( + `internal_id` Nullable(Int64), + `stable_id` Nullable(String), + `sample_type` Nullable(String), + `patient_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE sample_cna_event +( + `cna_event_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `genetic_profile_id` Nullable(Int64), + `annotation_json` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE sample_derived +( + `sample_unique_id` String, + `sample_unique_id_base64` String, + `sample_stable_id` String, + `patient_unique_id` String, + `patient_unique_id_base64` String, + `patient_stable_id` String, + `cancer_study_identifier` LowCardinality(String), + `internal_id` Int32, + `patient_internal_id` Int32, + `sample_type` String, + `sequenced` Int32, + `copy_number_segment_present` Int32 +) + ENGINE = MergeTree +ORDER BY (cancer_study_identifier, sample_unique_id); + +CREATE TABLE sample_list +( + `list_id` Nullable(Int64), + `stable_id` Nullable(String), + `category` Nullable(String), + `cancer_study_id` Nullable(Int64), + `name` Nullable(String), + `description` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE sample_list_list +( + `list_id` Nullable(Int64), + `sample_id` Nullable(Int64) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE sample_profile +( + `sample_id` Int64, + `genetic_profile_id` Int64, + `panel_id` Nullable(Int64) +) + ENGINE = ReplacingMergeTree +ORDER BY tuple(`sample_id`, `genetic_profile_id`); + +CREATE TABLE sample_to_gene_panel_derived +( + `sample_unique_id` String, + `alteration_type` LowCardinality(String), + `gene_panel_id` LowCardinality(String), + `cancer_study_identifier` LowCardinality(String), + `genetic_profile_id` LowCardinality(String) +) + ENGINE = MergeTree +ORDER BY (gene_panel_id, alteration_type, genetic_profile_id, sample_unique_id); + +CREATE TABLE structural_variant +( + `internal_id` Nullable(Int64), + `genetic_profile_id` Nullable(Int64), + `sample_id` Nullable(Int64), + `site1_entrez_gene_id` Nullable(Int64), + `site1_ensembl_transcript_id` Nullable(String), + `site1_chromosome` Nullable(String), + `site1_region` Nullable(String), + `site1_region_number` Nullable(Int64), + `site1_contig` Nullable(String), + `site1_position` Nullable(Int64), + `site1_description` Nullable(String), + `site2_entrez_gene_id` Nullable(Int64), + `site2_ensembl_transcript_id` Nullable(String), + `site2_chromosome` Nullable(String), + `site2_region` Nullable(String), + `site2_region_number` Nullable(Int64), + `site2_contig` Nullable(String), + `site2_position` Nullable(Int64), + `site2_description` Nullable(String), + `site2_effect_on_frame` Nullable(String), + `ncbi_build` Nullable(String), + `dna_support` Nullable(String), + `rna_support` Nullable(String), + `normal_read_count` Nullable(Int64), + `tumor_read_count` Nullable(Int64), + `normal_variant_count` Nullable(Int64), + `tumor_variant_count` Nullable(Int64), + `normal_paired_end_read_count` Nullable(Int64), + `tumor_paired_end_read_count` Nullable(Int64), + `normal_split_read_count` Nullable(Int64), + `tumor_split_read_count` Nullable(Int64), + `annotation` Nullable(String), + `breakpoint_type` Nullable(String), + `connection_type` Nullable(String), + `event_info` Nullable(String), + `class` Nullable(String), + `length` Nullable(Int64), + `comments` Nullable(String), + `sv_status` Nullable(String), + `annotation_json` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE type_of_cancer +( + `type_of_cancer_id` Nullable(String), + `name` Nullable(String), + `dedicated_color` Nullable(String), + `short_name` Nullable(String), + `parent` Nullable(String) +) + ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE users +( + `email` Nullable(String), + `name` Nullable(String), + `enabled` Nullable(Int32) +) + ENGINE = MergeTree +ORDER BY tuple(); diff --git a/src/test/resources/seed_mini.sql b/src/test/resources/seed_mini.sql index 19e9806a..98802b32 100644 --- a/src/test/resources/seed_mini.sql +++ b/src/test/resources/seed_mini.sql @@ -1,684 +1,553 @@ --- --- Copyright (c) 2015 - 2016 Memorial Sloan-Kettering Cancer Center. --- --- This library is distributed in the hope that it will be useful,but WITHOUT --- ANY WARRANTY,WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS --- FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder --- is on an "as is" basis,and Memorial Sloan-Kettering Cancer Center has no --- obligations to provide maintenance,support,updates,enhancements or --- modifications. In no event shall Memorial Sloan-Kettering Cancer Center be --- liable to any party for direct,indirect,special,incidental or --- consequential damages,including lost profits,arising out of the use of this --- software and its documentation,even if Memorial Sloan-Kettering Cancer --- Center has been advised of the possibility of such damage. --- --- This file is part of cBioPortal. --- --- cBioPortal is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation,either version 3 of the --- License. --- --- This program is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU Affero General Public License for more details. --- --- You should have received a copy of the GNU Affero General Public License --- along with this program. If not,see . --- ---------------------------------------------------------------------------- --- A manually extracted subset of data for a small number of genes and samples from the BRCA --- data set. This is intended to be used during unit testing,to validate the portal APIs. --- In theory,it should be enough to run up a portal. --- --- Prepared by Stuart Watt -- 13th May 2015 +/* +TRUNCATE TABLE IF EXISTS `structural_variant`; +TRUNCATE TABLE IF EXISTS `clinical_event_data`; +TRUNCATE TABLE IF EXISTS `clinical_event`; +TRUNCATE TABLE IF EXISTS `copy_number_seg_file`; +TRUNCATE TABLE IF EXISTS `copy_number_seg`; +TRUNCATE TABLE IF EXISTS `sample_cna_event`; +TRUNCATE TABLE IF EXISTS `cna_event`; +TRUNCATE TABLE IF EXISTS `gistic_to_gene`; +TRUNCATE TABLE IF EXISTS `gistic`; +TRUNCATE TABLE IF EXISTS `mut_sig`; +TRUNCATE TABLE IF EXISTS `clinical_attribute_meta`; +TRUNCATE TABLE IF EXISTS `mutation`; +TRUNCATE TABLE IF EXISTS `mutation_event`; +TRUNCATE TABLE IF EXISTS `sample_profile`; +TRUNCATE TABLE IF EXISTS `gene_panel`; +TRUNCATE TABLE IF EXISTS `gene_panel_list`; +TRUNCATE TABLE IF EXISTS `genetic_profile_samples`; +TRUNCATE TABLE IF EXISTS `genetic_alteration`; +TRUNCATE TABLE IF EXISTS `genetic_profile`; +TRUNCATE TABLE IF EXISTS `gene_alias`; +TRUNCATE TABLE IF EXISTS `gene`; +TRUNCATE TABLE IF EXISTS `clinical_sample`; +TRUNCATE TABLE IF EXISTS `sample`; +TRUNCATE TABLE IF EXISTS `sample_list_list`; +TRUNCATE TABLE IF EXISTS `sample_list`; +TRUNCATE TABLE IF EXISTS `clinical_patient`; +TRUNCATE TABLE IF EXISTS `patient`; +TRUNCATE TABLE IF EXISTS `authorities`; +TRUNCATE TABLE IF EXISTS `users`; +TRUNCATE TABLE IF EXISTS `cancer_study`; +TRUNCATE TABLE IF EXISTS `type_of_cancer`; +TRUNCATE TABLE IF EXISTS `genetic_entity`; +*/ -SET SESSION sql_mode = 'ANSI_QUOTES'; +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('acc','Adrenocortical Carcinoma','Purple','ACC','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('acbc','Adenoid Cystic Breast Cancer','HotPink','ACBC','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('acyc','Adenoid Cystic Carcinoma','DarkRed','ACyC','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('aeca','Sweat Gland Carcinoma (Apocrine Eccrine Carcinoma)','Black','AECA','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('afx','Atypical Fibroxanthoma','Black','AFX','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('all','Pediatric Acute Lymphoid Leukemia','LightSalmon','ALL','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('aml','Acute Monocytic Leukemia','LightSalmon','AML-M5','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('blad','Bladder Urothelial Adenocarcinoma','Yellow','Bladder','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('blca','Bladder Urothelial Carcinoma','Yellow','Bladder','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('bpdcn','Blastic Plasmacytoid Dendritic Cell Neoplasm','LightSalmon','BPDCN','tissue'); +INSERT INTO `type_of_cancer` (`type_of_cancer_id`, `name`, `dedicated_color`, `short_name`, `parent`) VALUES ('brca','Breast Invasive Carcinoma','HotPink','Breast','tissue'); -DELETE FROM structural_variant; -DELETE FROM clinical_event_data; -DELETE FROM clinical_event; -DELETE FROM copy_number_seg_file; -DELETE FROM copy_number_seg; -DELETE FROM sample_cna_event; -DELETE FROM cna_event; -DELETE FROM gistic_to_gene; -DELETE FROM gistic; -DELETE FROM mut_sig; -DELETE FROM clinical_attribute_meta; -DELETE FROM mutation; -DELETE FROM mutation_event; -DELETE FROM sample_profile; -DELETE FROM gene_panel; -DELETE FROM gene_panel_list; -DELETE FROM genetic_profile_samples; -DELETE FROM genetic_alteration; -DELETE FROM genetic_profile; -DELETE FROM gene_alias; -DELETE FROM gene; -DELETE FROM clinical_sample; -DELETE FROM sample; -DELETE FROM sample_list_list; -DELETE FROM sample_list; -DELETE FROM clinical_patient; -DELETE FROM patient; -DELETE FROM authorities; -DELETE FROM users; -DELETE FROM cancer_study; -DELETE FROM type_of_cancer; -DELETE FROM genetic_entity; - --- type_of_cancer -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('acc','Adrenocortical Carcinoma','Purple','ACC','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('acbc','Adenoid Cystic Breast Cancer','HotPink','ACBC','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('acyc','Adenoid Cystic Carcinoma','DarkRed','ACyC','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('aeca','Sweat Gland Carcinoma (Apocrine Eccrine Carcinoma)','Black','AECA','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('afx','Atypical Fibroxanthoma','Black','AFX','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('all','Pediatric Acute Lymphoid Leukemia','LightSalmon','ALL','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('aml','Acute Monocytic Leukemia','LightSalmon','AML-M5','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('blad','Bladder Urothelial Adenocarcinoma','Yellow','Bladder','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('blca','Bladder Urothelial Carcinoma','Yellow','Bladder','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('bpdcn','Blastic Plasmacytoid Dendritic Cell Neoplasm','LightSalmon','BPDCN','tissue'); -INSERT INTO "type_of_cancer" ("TYPE_OF_CANCER_ID","NAME","DEDICATED_COLOR","SHORT_NAME","PARENT") VALUES ('brca','Breast Invasive Carcinoma','HotPink','Breast','tissue'); - --- reference_genome INSERT INTO `reference_genome` VALUES (1,'human','hg19','GRCh37',2897310462,'http://hgdownload.cse.ucsc.edu/goldenPath/hg19/bigZips','2009-02-01'); INSERT INTO `reference_genome` VALUES (2,'human','hg38','GRCh38',3049315783,'http://hgdownload.cse.ucsc.edu/goldenPath/hg38/bigZips','2013-12-01'); --- cancer_study -INSERT INTO "cancer_study" ("CANCER_STUDY_ID","CANCER_STUDY_IDENTIFIER","TYPE_OF_CANCER_ID","NAME","DESCRIPTION","PUBLIC","PMID","CITATION","GROUPS","REFERENCE_GENOME_ID") -VALUES (1,'study_tcga_pub','brca','Breast Invasive Carcinoma (TCGA,Nature 2012)','The Cancer Genome Atlas (TCGA) Breast Invasive Carcinoma project. 825 cases.
Nature 2012. Raw data via the TCGA Data Portal.',1,'23000897,26451490','TCGA,Nature 2012,...','SU2C-PI3K;PUBLIC;GDAC',1); - --- clinical_attribute_meta -INSERT INTO "clinical_attribute_meta" -VALUES ('DFS_MONTHS','Disease Free (Months)','Disease free in months since treatment','NUMBER',0,'1',1), -('DFS_STATUS','Disease Free Status','Disease free status','STRING',0,'1',1), -('OS_MONTHS','Overall Survival (Months)','Overall survival in months since diagnosis','NUMBER',0,'1',1), -('OS_STATUS','Overall Survival Status','Overall survival status','STRING',0,'1',1), -('SAMPLE_COUNT','Number of Samples Per Patient','Number of Samples Per Patient','STRING',1,'1',1), -('SUBTYPE','Subtype','Subtype description','STRING',0,'1',1); - --- gene as genetic_entity -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,207,'AKT1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,208,'AKT2','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,10000,'AKT3','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,369,'ARAF','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,472,'ATM','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,673,'BRAF','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,672,'BRCA1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,675,'BRCA2','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,3265,'HRAS','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,3845,'KRAS','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,4893,'NRAS','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,51259,'TMEM216','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,282770,'OR10AG1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,983,'CDK1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,8085,'KMT2D','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); --- add genes for structural variant events -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,57670,'KIAA1549','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,27436,'EML4','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,238,'ALK','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,2115,'ETV1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,7273,'TTN','protein-coding'); - --- for test panels 1 and 2 -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,3983,'ABLIM1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,80070,'ADAMTS20','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,253559,'CADM2','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,1838,'DTNB','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,2648,'KAT2A','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,4437,'MSH3','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,4602,'MYB','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,440348,'NPIPB15','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,56914,'OTOR','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,27334,'P2RY10','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,9780,'PIEZO1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,2261,'FGFR3','protein-coding'); +INSERT INTO `cancer_study` (`cancer_study_id`, `cancer_study_identifier`, `type_of_cancer_id`, `name`, `description`, `public`, `pmid`, `citation`, `groups`, `reference_genome_id`) VALUES (1,'study_tcga_pub','brca','Breast Invasive Carcinoma (TCGA,Nature 2012)','The Cancer Genome Atlas (TCGA) Breast Invasive Carcinoma project. 825 cases.
Nature 2012. Raw data via the TCGA Data Portal.',1,'23000897,26451490','TCGA,Nature 2012,...','SU2C-PI3K;PUBLIC;GDAC',1); --- missing genes for study_es_0 --- additional genes for CNA data -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,116983,'ACAP3','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,2073,'ERCC5','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,219293,'ATAD3C','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,375790,'AGRN','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,54998,'AURKAIP1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,55210,'ATAD3A','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,83858,'ATAD3B','protein-coding'); --- genes for data_methylation_hm27.txt -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,24145,'PANX1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,283234,'CCDC88B','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,3232,'HOXD3','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,3613,'IMPA2','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,389,'RHOC','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,487,'ATP2A1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,7871,'SLMAP','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,8148,'TAF15','protein-coding'); --- gene panels -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,55,'ACP3','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,81061,'OR11H1','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,388946,'TMEM247','protein-coding'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE") VALUES ('GENE'); -SET @max_entity_id = (Select MAX(ID) from genetic_entity); -INSERT INTO "gene" ("GENETIC_ENTITY_ID","ENTREZ_GENE_ID","HUGO_GENE_SYMBOL","TYPE") VALUES (@max_entity_id,7157,'TP53','protein-coding'); +INSERT INTO `clinical_attribute_meta` VALUES ('DFS_MONTHS','Disease Free (Months)','Disease free in months since treatment','NUMBER',0,'1',1), ('DFS_STATUS','Disease Free Status','Disease free status','STRING',0,'1',1), ('OS_MONTHS','Overall Survival (Months)','Overall survival in months since diagnosis','NUMBER',0,'1',1), ('OS_STATUS','Overall Survival Status','Overall survival status','STRING',0,'1',1), ('SAMPLE_COUNT','Number of Samples Per Patient','Number of Samples Per Patient','STRING',1,'1',1), ('SUBTYPE','Subtype','Subtype description','STRING',0,'1',1); --- Generic genetic entities -INSERT INTO "genetic_entity" ("ENTITY_TYPE", "STABLE_ID") VALUES ('GENERIC_ASSAY', 'Erlotinib'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE", "STABLE_ID") VALUES ('GENERIC_ASSAY', 'Irinotecan'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE", "STABLE_ID") VALUES ('GENERIC_ASSAY', 'L-685458'); -INSERT INTO "genetic_entity" ("ENTITY_TYPE", "STABLE_ID") VALUES ('GENERIC_ASSAY', 'Lapatinib'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (1,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (1,207,'AKT1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (2,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (2,208,'AKT2','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (3,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (3,10000,'AKT3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (4,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (4,369,'ARAF','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (5,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (5,472,'ATM','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (6,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (6,673,'BRAF','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (7,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (7,672,'BRCA1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (8,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (8,675,'BRCA2','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (9,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (9,3265,'HRAS','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (10,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (10,3845,'KRAS','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (11,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (11,4893,'NRAS','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (12,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (12,51259,'TMEM216','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (13,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (13,282770,'OR10AG1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (14,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (14,983,'CDK1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (15,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (15,8085,'KMT2D','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (16,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (16,57670,'KIAA1549','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (17,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (17,27436,'EML4','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (18,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (18,238,'ALK','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (19,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (19,2115,'ETV1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (20,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (20,7273,'TTN','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (21,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (21,3983,'ABLIM1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (22,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (22,80070,'ADAMTS20','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (23,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (23,253559,'CADM2','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (24,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (24,1838,'DTNB','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (25,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (25,2648,'KAT2A','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (26,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (26,4437,'MSH3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (27,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (27,4602,'MYB','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (28,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (28,440348,'NPIPB15','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (29,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (29,56914,'OTOR','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (30,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (30,27334,'P2RY10','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (31,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (31,9780,'PIEZO1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (32,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (32,2261,'FGFR3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (33,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (33,116983,'ACAP3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (34,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (34,2073,'ERCC5','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (35,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (35,219293,'ATAD3C','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (36,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (36,375790,'AGRN','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (37,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (37,54998,'AURKAIP1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (38,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (38,55210,'ATAD3A','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (39,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (39,83858,'ATAD3B','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (40,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (40,24145,'PANX1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (41,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (41,283234,'CCDC88B','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (42,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (42,3232,'HOXD3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (43,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (43,3613,'IMPA2','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (44,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (44,389,'RHOC','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (45,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (45,487,'ATP2A1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (46,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (46,7871,'SLMAP','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (47,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (47,8148,'TAF15','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (48,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (48,55,'ACP3','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (49,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (49,81061,'OR11H1','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (50,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (50,388946,'TMEM247','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`) VALUES (51,'GENE'); +INSERT INTO `gene` (`genetic_entity_id`, `entrez_gene_id`, `hugo_gene_symbol`, `type`) VALUES (51,7157,'TP53','protein-coding'); +INSERT INTO `genetic_entity` (`id`, `entity_type`, `stable_id`) VALUES (52,'GENERIC_ASSAY', 'Erlotinib'); +INSERT INTO `genetic_entity` (`id`, `entity_type`, `stable_id`) VALUES (53,'GENERIC_ASSAY', 'Irinotecan'); +INSERT INTO `genetic_entity` (`id`, `entity_type`, `stable_id`) VALUES (54,'GENERIC_ASSAY', 'L-685458'); +INSERT INTO `genetic_entity` (`id`, `entity_type`, `stable_id`) VALUES (55,'GENERIC_ASSAY', 'Lapatinib'); --- cna_event -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (20093,207,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (20092,207,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (26161,208,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (2774,10000,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (16668,472,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (16669,472,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (11284,673,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (11283,673,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (23934,672,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (23933,672,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (18460,675,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (18459,675,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (15117,3265,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (15118,3265,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (17280,3845,2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (1678,4893,-2); -INSERT INTO "cna_event" ("CNA_EVENT_ID","ENTREZ_GENE_ID","ALTERATION") VALUES (1677,4893,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (20093,207,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (20092,207,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (26161,208,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (2774,10000,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (16668,472,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (16669,472,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (11284,673,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (11283,673,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (23934,672,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (23933,672,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (18460,675,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (18459,675,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (15117,3265,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (15118,3265,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (17280,3845,2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (1678,4893,-2); +INSERT INTO `cna_event` (`cna_event_id`, `entrez_gene_id`, `alteration`) VALUES (1677,4893,2); --- gene_alias -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (207,'AKT'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (207,'CWS6'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (208,'HIHGHH'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (208,'PKBB'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (10000,'MPPH'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (10000,'MPPH2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (369,'A-RAF'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (369,'ARAF1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (472,'AT1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (472,'ATA'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (673,'B-RAF1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (673,'BRAF1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (672,'BRCAI'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (672,'BRCC1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (675,'BRCC2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (675,'BROVCA2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (675,'FACD'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3265,'C-BAS/HAS'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3265,'C-H-RAS'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3845,'C-K-RAS'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3845,'KRAS1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3845,'KRAS2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4893,'N-ras'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4893,'NCMS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (207,'AKT'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (207,'CWS6'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (208,'HIHGHH'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (208,'PKBB'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (10000,'MPPH'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (10000,'MPPH2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (369,'A-RAF'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (369,'ARAF1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (472,'AT1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (472,'ATA'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (673,'B-RAF1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (673,'BRAF1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (672,'BRCAI'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (672,'BRCC1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (675,'BRCC2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (675,'BROVCA2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (675,'FACD'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3265,'C-BAS/HAS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3265,'C-H-RAS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3845,'C-K-RAS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3845,'KRAS1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3845,'KRAS2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4893,'N-ras'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4893,'NCMS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (56914,'FDP'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (56914,'MIAL1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (253559,'IGSF4D'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (253559,'Necl-3'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (253559,'NECL3'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (253559,'SynCAM 2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (253559,'synCAM2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4437,'MRP1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4437,'DUP'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4437,'FAP4'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (27334,'P2Y10'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (27334,'LYPSR2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (80070,'ADAM-TS20'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (80070,'GON-1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (80070,'ADAMTS-20'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2648,'hGCN5'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2648,'GCN5'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2648,'GCN5L2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2648,'PCAF-b'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4602,'c-myb_CDS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4602,'Cmyb'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4602,'c-myb'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (4602,'efg'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (440348,'A-761H5.4'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (440348,'NPIPL2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3983,'LIMAB1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3983,'abLIM-1'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3983,'ABLIM'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (3983,'LIMATIN'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (9780,'FAM38A'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (9780,'Mib'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (9780,'LMPH3'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (9780,'DHS'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2261,'JTK4'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2261,'HSFGFR3EX'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2261,'ACH'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2261,'CEK2'); +INSERT INTO `gene_alias` (`entrez_gene_id`, `gene_alias`) VALUES (2261,'CD333'); --- for test panels 1 and 2 -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (56914,'FDP'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (56914,'MIAL1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (253559,'IGSF4D'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (253559,'Necl-3'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (253559,'NECL3'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (253559,'SynCAM 2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (253559,'synCAM2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4437,'MRP1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4437,'DUP'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4437,'FAP4'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (27334,'P2Y10'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (27334,'LYPSR2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (80070,'ADAM-TS20'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (80070,'GON-1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (80070,'ADAMTS-20'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2648,'hGCN5'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2648,'GCN5'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2648,'GCN5L2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2648,'PCAF-b'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4602,'c-myb_CDS'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4602,'Cmyb'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4602,'c-myb'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (4602,'efg'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (440348,'A-761H5.4'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (440348,'NPIPL2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3983,'LIMAB1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3983,'abLIM-1'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3983,'ABLIM'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (3983,'LIMATIN'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (9780,'FAM38A'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (9780,'Mib'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (9780,'LMPH3'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (9780,'DHS'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2261,'JTK4'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2261,'HSFGFR3EX'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2261,'ACH'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2261,'CEK2'); -INSERT INTO "gene_alias" ("ENTREZ_GENE_ID","GENE_ALIAS") VALUES (2261,'CD333'); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(207,'14q32.33',105235686,105262088,14,1); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(207,'14q32.33',104769349,104795751,14,2); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(208,'19q13.2',40736224,40791443,19,1); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(208,'19q13.2',40230317,40285536,19,2); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(51259,'11q12.2',61159159,61166335,11,1); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(51259,'11q12.2',61391687,61398863,11,2); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(282770,'11q12.1',55734975,55735990,11,1); +INSERT INTO reference_genome_gene (`entrez_gene_id`, `cytoband`, `start`, `end`, `chr`, `reference_genome_id`) VALUES(282770,'11q12.1',55967558,55968463,11,2); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(207,'14q32.33',105235686,105262088,14,1); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(207,'14q32.33',104769349,104795751,14,2); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(208,'19q13.2',40736224,40791443,19,1); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(208,'19q13.2',40230317,40285536,19,2); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(51259,'11q12.2',61159159,61166335,11,1); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(51259,'11q12.2',61391687,61398863,11,2); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(282770,'11q12.1',55734975,55735990,11,1); -INSERT INTO reference_genome_gene (ENTREZ_GENE_ID,CYTOBAND,START,END,CHR,REFERENCE_GENOME_ID) VALUES(282770,'11q12.1',55967558,55968463,11,2); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (2,'study_tcga_pub_gistic',1,'COPY_NUMBER_ALTERATION','DISCRETE','Putative copy-number alterations from GISTIC','Putative copy-number from GISTIC 2.0. Values: -2 = homozygous deletion; -1 = hemizygous deletion; 0 = neutral / no change; 1 = gain; 2 = high level amplification.','1'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (3,'study_tcga_pub_mrna',1,'MRNA_EXPRESSION','Z-SCORE','mRNA expression (microarray)','Expression levels (Agilent microarray).','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (4,'study_tcga_pub_log2CNA',1,'COPY_NUMBER_ALTERATION','LOG2-VALUE','Log2 copy-number values','Log2 copy-number values for each gene (from Affymetrix SNP6).','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (5,'study_tcga_pub_methylation_hm27',1,'METHYLATION','CONTINUOUS','Methylation (HM27)','Methylation beta-values (HM27 platform). For genes with multiple methylation probes,the probe least correlated with expression is selected.','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (6,'study_tcga_pub_mutations',1,'MUTATION_EXTENDED','MAF','Mutations','Mutation data from whole exome sequencing.','1'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (7,'study_tcga_pub_structural_variants',1,'STRUCTURAL_VARIANT','SV','Structural Variants','Structural Variants test data.','1'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (8,'study_tcga_pub_cna_long',1,'COPY_NUMBER_ALTERATION','DISCRETE_LONG','CNA values','CNA values dummy data','1'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (9,'study_tcga_pub_rppa',1,'PROTEIN_LEVEL','LOG2-VALUE','RPPA values','RPPA values dummy data','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (10,'study_tcga_pub_treatment_ic50',1,'GENERIC_ASSAY','LIMIT-VALUE','test treatment values','treatment values dummy data','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (11,'study_tcga_pub_gsva_scores',1,'GENESET_SCORE','GSVA-SCORE','test gsva score values','test gsva score description','0'); +INSERT INTO `genetic_profile` (`genetic_profile_id`, `stable_id`, `cancer_study_id`, `genetic_alteration_type`, `datatype`, `name`, `description`, `show_profile_in_analysis_tab`) VALUES (12,'study_tcga_pub_gsva_pvalues',1,'GENESET_SCORE','P-VALUE','test gsva p-values','test gsva p-values description','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (2,'study_tcga_pub_gistic',1,'COPY_NUMBER_ALTERATION','DISCRETE','Putative copy-number alterations from GISTIC','Putative copy-number from GISTIC 2.0. Values: -2 = homozygous deletion; -1 = hemizygous deletion; 0 = neutral / no change; 1 = gain; 2 = high level amplification.','1'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (3,'study_tcga_pub_mrna',1,'MRNA_EXPRESSION','Z-SCORE','mRNA expression (microarray)','Expression levels (Agilent microarray).','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (4,'study_tcga_pub_log2CNA',1,'COPY_NUMBER_ALTERATION','LOG2-VALUE','Log2 copy-number values','Log2 copy-number values for each gene (from Affymetrix SNP6).','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (5,'study_tcga_pub_methylation_hm27',1,'METHYLATION','CONTINUOUS','Methylation (HM27)','Methylation beta-values (HM27 platform). For genes with multiple methylation probes,the probe least correlated with expression is selected.','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (6,'study_tcga_pub_mutations',1,'MUTATION_EXTENDED','MAF','Mutations','Mutation data from whole exome sequencing.','1'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (7,'study_tcga_pub_structural_variants',1,'STRUCTURAL_VARIANT','SV','Structural Variants','Structural Variants test data.','1'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (8,'study_tcga_pub_cna_long',1,'COPY_NUMBER_ALTERATION','DISCRETE_LONG','CNA values','CNA values dummy data','1'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (9,'study_tcga_pub_rppa',1,'PROTEIN_LEVEL','LOG2-VALUE','RPPA values','RPPA values dummy data','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (10,'study_tcga_pub_treatment_ic50',1,'GENERIC_ASSAY','LIMIT-VALUE','test treatment values','treatment values dummy data','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (11,'study_tcga_pub_gsva_scores',1,'GENESET_SCORE','GSVA-SCORE','test gsva score values','test gsva score description','0'); -INSERT INTO "genetic_profile" ("GENETIC_PROFILE_ID","STABLE_ID","CANCER_STUDY_ID","GENETIC_ALTERATION_TYPE","DATATYPE","NAME","DESCRIPTION","SHOW_PROFILE_IN_ANALYSIS_TAB") VALUES (12,'study_tcga_pub_gsva_pvalues',1,'GENESET_SCORE','P-VALUE','test gsva p-values','test gsva p-values description','0'); +INSERT INTO gene_panel (`internal_id`, `stable_id`, `description`) VALUES (1,'TESTPANEL_CNA_DISCRETE_LONG_FORMAT','Some test panel'); +INSERT INTO gene_panel (`internal_id`, `stable_id`, `description`) VALUES (2,'TSTGNPNLCNADS','The CNA Discrete test panel'); +INSERT INTO gene_panel (`internal_id`, `stable_id`, `description`) VALUES (3,'TSTGNPNLMUTEXT','The mutation extended test panel'); +INSERT INTO gene_panel (`internal_id`, `stable_id`, `description`) VALUES (4,'TSTGNPNLGENASS','The generic assay test panel'); +INSERT INTO gene_panel (`internal_id`, `stable_id`, `description`) VALUES (5,'TSTGNPNLSV','The structural variance test panel'); --- gene_panel -INSERT INTO gene_panel (INTERNAL_ID,STABLE_ID,DESCRIPTION) VALUES (1,'TESTPANEL_CNA_DISCRETE_LONG_FORMAT','Some test panel'); -INSERT INTO gene_panel (INTERNAL_ID,STABLE_ID,DESCRIPTION) VALUES (2,'TSTGNPNLCNADS','The CNA Discrete test panel'); -INSERT INTO gene_panel (INTERNAL_ID,STABLE_ID,DESCRIPTION) VALUES (3,'TSTGNPNLMUTEXT','The mutation extended test panel'); -INSERT INTO gene_panel (INTERNAL_ID,STABLE_ID,DESCRIPTION) VALUES (4,'TSTGNPNLGENASS','The generic assay test panel'); -INSERT INTO gene_panel (INTERNAL_ID,STABLE_ID,DESCRIPTION) VALUES (5,'TSTGNPNLSV','The structural variance test panel'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,3,'0,0,1,2,0,1,1,1,0,1,1,1,0,1,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,1,'0,0,0,0,0,0,0,0,-1,0,0,0,-1,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,2,'0,0,0,1,0,0,0,1,0,0,0,2,0,2,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,9,'0,0,0,0,0,0,1,0,0,0,0,-1,0,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,10,'0,0,0,1,0,0,-1,1,1,0,1,2,2,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,5,'0,-1,-1,0,0,-1,0,1,0,-1,-1,1,-1,-1,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,11,'0,-1,0,-1,0,0,0,0,-1,1,-1,0,0,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,7,'0,0,0,0,0,1,0,-1,0,-1,-1,0,-1,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,6,'0,0,0,1,0,0,0,0,0,0,-1,0,0,0,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (2,8,'0,-1,0,0,0,0,-1,0,1,0,-1,1,-1,-1,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,3,'-0.473833333333333,1.51866666666667,0.148333333333333,-0.187666666666667,0.914,-0.664333333333333,-1.70783333333333,0.976,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,1,'-1.102375,-0.24375,0.018625,-0.157,0.33075,1.008,0.68175,-0.664875,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,2,'-1.22235714285714,-0.592571428571429,-0.176642857142857,-0.310428571428571,-1.19892857142857,-0.670142857142857,0.0779285714285714,-0.302642857142857,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,9,'0.068,-0.062,-0.167833333333333,0.511666666666667,2.02066666666667,0.389166666666667,-0.724666666666666,0.9485,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,4,'-1.12475,-0.30675,0.1805,-0.59775,0.16625,0.402,0.243,-0.996,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,10,'-0.17075,0.4045,0.185333333333333,0.4285,1.67616666666667,0.238,0.469833333333333,2.15883333333333,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,5,'-1.50341666666667,-1.92183333333333,-1.75541666666667,-1.57325,-1.02958333333333,-1.39791666666667,-1.51483333333333,-2.07091666666667,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,11,'-1.91125,-2.0595,-1.22825,-1.319,-4.16675,-1.187875,0.280625,-0.13075,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,7,'-1.66108333333333,-1.38791666666667,-1.92483333333333,-1.65625,-0.35825,-1.99566666666667,-0.136416666666667,-0.70975,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,6,'0.2305,0.56,-0.10225,-0.0855,-0.012,0.138,0.141,0.6095,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (3,8,'-0.57075,-1.3405,-1.541,-0.40475,0.629,-1.2315,0.768,-0.033,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,3,'0.017,0.032,0.872,0.704,0.009,0.485,0.848,0.685,-0.270,1.054,0.658,0.455,0.256,0.628,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,1,'-0.005,-0.003,-0.022,0.072,-0.042,0.015,0.045,0.026,-0.441,-0.030,0.009,0.154,-0.348,0.011,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,2,'0.045,-0.048,0.043,0.324,-0.064,0.036,0.147,0.404,0.112,0.012,0.043,1.418,0.069,1.670,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,9,'0.003,-0.043,0.009,0.009,-0.043,0.031,0.411,-0.048,0.049,-0.030,0.056,-0.695,-0.022,-0.001,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,10,'-0.009,0.001,0.006,0.379,0.022,-0.010,-0.473,1.172,2.823,0.014,0.339,1.450,3.160,0.026,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,5,'-0.006,-0.500,-0.462,-0.013,-0.003,-0.448,0.135,0.522,0.211,-0.578,-0.350,0.655,-0.449,-0.668,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,11,'0.011,-0.420,0.014,-0.341,-0.019,0.020,0.032,-0.052,-0.453,0.455,-0.318,0.278,0.240,0.034,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,7,'0.044,-0.082,0.027,-0.003,-0.012,0.486,-0.300,-0.397,-0.284,-0.547,-0.301,0.218,-0.389,-0.011,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,6,'0.015,0.005,0.005,0.344,-0.011,0.171,0.140,-0.013,0.021,0.010,-0.385,-0.177,0.075,0.002,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (4,8,'-0.005,-0.502,-0.240,0.016,0.022,-0.029,-0.319,0.004,1.020,-0.024,-0.366,0.768,-0.397,-0.667,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,3,'0.841525757,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,1,'0.134488708,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,2,'0.072712077,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,9,'0.039372,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,4,'0.125441004,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,10,'0.049338479,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,5,'0.025402093,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,11,'0.105977072,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,7,'0.066638638,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,6,'0.020369562,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (5,8,'0.793930197,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,3,'-0.472,1.514,0.145,-0.183,0.913,-0.665,-1.700,0.976,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,1,'-1.102,-0.243,0.018,-0.154,0.330,1.005,0.681,-0.664,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,2,'-1.221,-0.592,-0.176,-0.310,-1.198,-0.670,0.077,-0.302,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,9,'0.061,-0.055,-0.165,0.517,2.021,0.381,-0.728,0.944,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,4,'-1.129,-0.306,0.180,-0.601,0.166,0.402,0.243,-0.999,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,10,'0.177,0.404,0.188,0.428,1.676,0.238,0.469,2.161,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,5,'-1.503,-1.925,-1.755,-1.576,-1.029,-1.401,-1.514,-2.074,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,11,'-1.914,-2.059,-1.228,-1.322,-4.166,-1.187,0.284,-0.130,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,7,'-1.661,-1.392,-1.924,-1.656,-0.361,-1.998,-0.136,-0.709,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,6,'0.233,0.561,-0.106,-0.085,-0.012,0.143,0.141,0.609,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (9,8,'-0.570,-1.340,-1.544,-0.404,0.632,-1.231,0.771,-0.036,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (10,52,'5.2,>8,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (10,53,'>8,7.1,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (10,54,'>4.6,7.2,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (10,55,'6.9,>~8,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (11,8,'0.1,'); +INSERT INTO `genetic_alteration` (`genetic_profile_id`, `genetic_entity_id`, `values`) VALUES (12,8,'0.2,'); --- genetic_alteration -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 10000),'0,0,1,2,0,1,1,1,0,1,1,1,0,1,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 207),'0,0,0,0,0,0,0,0,-1,0,0,0,-1,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 208),'0,0,0,1,0,0,0,1,0,0,0,2,0,2,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3265),'0,0,0,0,0,0,1,0,0,0,0,-1,0,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3845),'0,0,0,1,0,0,-1,1,1,0,1,2,2,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 472),'0,-1,-1,0,0,-1,0,1,0,-1,-1,1,-1,-1,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 4893),'0,-1,0,-1,0,0,0,0,-1,1,-1,0,0,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 672),'0,0,0,0,0,1,0,-1,0,-1,-1,0,-1,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 673),'0,0,0,1,0,0,0,0,0,0,-1,0,0,0,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (2,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'0,-1,0,0,0,0,-1,0,1,0,-1,1,-1,-1,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 10000),'-0.473833333333333,1.51866666666667,0.148333333333333,-0.187666666666667,0.914,-0.664333333333333,-1.70783333333333,0.976,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 207),'-1.102375,-0.24375,0.018625,-0.157,0.33075,1.008,0.68175,-0.664875,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 208),'-1.22235714285714,-0.592571428571429,-0.176642857142857,-0.310428571428571,-1.19892857142857,-0.670142857142857,0.0779285714285714,-0.302642857142857,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3265),'0.068,-0.062,-0.167833333333333,0.511666666666667,2.02066666666667,0.389166666666667,-0.724666666666666,0.9485,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 369),'-1.12475,-0.30675,0.1805,-0.59775,0.16625,0.402,0.243,-0.996,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3845),'-0.17075,0.4045,0.185333333333333,0.4285,1.67616666666667,0.238,0.469833333333333,2.15883333333333,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 472),'-1.50341666666667,-1.92183333333333,-1.75541666666667,-1.57325,-1.02958333333333,-1.39791666666667,-1.51483333333333,-2.07091666666667,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 4893),'-1.91125,-2.0595,-1.22825,-1.319,-4.16675,-1.187875,0.280625,-0.13075,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 672),'-1.66108333333333,-1.38791666666667,-1.92483333333333,-1.65625,-0.35825,-1.99566666666667,-0.136416666666667,-0.70975,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 673),'0.2305,0.56,-0.10225,-0.0855,-0.012,0.138,0.141,0.6095,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (3,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'-0.57075,-1.3405,-1.541,-0.40475,0.629,-1.2315,0.768,-0.033,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 10000),'0.017,0.032,0.872,0.704,0.009,0.485,0.848,0.685,-0.270,1.054,0.658,0.455,0.256,0.628,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 207),'-0.005,-0.003,-0.022,0.072,-0.042,0.015,0.045,0.026,-0.441,-0.030,0.009,0.154,-0.348,0.011,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 208),'0.045,-0.048,0.043,0.324,-0.064,0.036,0.147,0.404,0.112,0.012,0.043,1.418,0.069,1.670,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3265),'0.003,-0.043,0.009,0.009,-0.043,0.031,0.411,-0.048,0.049,-0.030,0.056,-0.695,-0.022,-0.001,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3845),'-0.009,0.001,0.006,0.379,0.022,-0.010,-0.473,1.172,2.823,0.014,0.339,1.450,3.160,0.026,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 472),'-0.006,-0.500,-0.462,-0.013,-0.003,-0.448,0.135,0.522,0.211,-0.578,-0.350,0.655,-0.449,-0.668,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 4893),'0.011,-0.420,0.014,-0.341,-0.019,0.020,0.032,-0.052,-0.453,0.455,-0.318,0.278,0.240,0.034,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 672),'0.044,-0.082,0.027,-0.003,-0.012,0.486,-0.300,-0.397,-0.284,-0.547,-0.301,0.218,-0.389,-0.011,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 673),'0.015,0.005,0.005,0.344,-0.011,0.171,0.140,-0.013,0.021,0.010,-0.385,-0.177,0.075,0.002,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (4,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'-0.005,-0.502,-0.240,0.016,0.022,-0.029,-0.319,0.004,1.020,-0.024,-0.366,0.768,-0.397,-0.667,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 10000),'0.841525757,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 207),'0.134488708,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 208),'0.072712077,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3265),'0.039372,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 369),'0.125441004,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3845),'0.049338479,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 472),'0.025402093,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 4893),'0.105977072,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 672),'0.066638638,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 673),'0.020369562,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (5,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'0.793930197,'); --- RPPA -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 10000),'-0.472,1.514,0.145,-0.183,0.913,-0.665,-1.700,0.976,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 207),'-1.102,-0.243,0.018,-0.154,0.330,1.005,0.681,-0.664,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 208),'-1.221,-0.592,-0.176,-0.310,-1.198,-0.670,0.077,-0.302,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3265),'0.061,-0.055,-0.165,0.517,2.021,0.381,-0.728,0.944,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 369),'-1.129,-0.306,0.180,-0.601,0.166,0.402,0.243,-0.999,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 3845),'0.177,0.404,0.188,0.428,1.676,0.238,0.469,2.161,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 472),'-1.503,-1.925,-1.755,-1.576,-1.029,-1.401,-1.514,-2.074,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 4893),'-1.914,-2.059,-1.228,-1.322,-4.166,-1.187,0.284,-0.130,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 672),'-1.661,-1.392,-1.924,-1.656,-0.361,-1.998,-0.136,-0.709,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 673),'0.233,0.561,-0.106,-0.085,-0.012,0.143,0.141,0.609,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (9,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'-0.570,-1.340,-1.544,-0.404,0.632,-1.231,0.771,-0.036,'); --- Generic assay data -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (10,(Select "ID" from "genetic_entity" where "STABLE_ID" = 'Erlotinib'),'5.2,>8,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (10,(Select "ID" from "genetic_entity" where "STABLE_ID" = 'Irinotecan'),'>8,7.1,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (10,(Select "ID" from "genetic_entity" where "STABLE_ID" = 'L-685458'),'>4.6,7.2,'); -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (10,(Select "ID" from "genetic_entity" where "STABLE_ID" = 'Lapatinib'),'6.9,>~8,'); --- GSVA score -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (11,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'0.1,'); --- GSVA p-value -INSERT INTO "genetic_alteration" ("GENETIC_PROFILE_ID","GENETIC_ENTITY_ID","VALUES") VALUES (12,(Select "GENETIC_ENTITY_ID" from "gene" where "ENTREZ_GENE_ID" = 675),'0.2,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (2,'1,2,3,4,5,6,7,8,9,10,11,12,13,14,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (3,'2,3,6,8,9,10,12,13,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (4,'1,2,3,4,5,6,7,8,9,10,11,12,13,14,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (5,'2,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (9,'2,3,6,8,9,10,12,13,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (10,'2,3,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (11,'22,'); +INSERT INTO `genetic_profile_samples` (`genetic_profile_id`, `ordered_sample_list`) VALUES (12,'23,'); --- genetic_profile_samples -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (2,'1,2,3,4,5,6,7,8,9,10,11,12,13,14,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (3,'2,3,6,8,9,10,12,13,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (4,'1,2,3,4,5,6,7,8,9,10,11,12,13,14,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (5,'2,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (9,'2,3,6,8,9,10,12,13,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (10,'2,3,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (11,'22,'); -INSERT INTO "genetic_profile_samples" ("GENETIC_PROFILE_ID","ORDERED_SAMPLE_LIST") VALUES (12,'23,'); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (1,'TCGA-A1-A0SB',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (2,'TCGA-A1-A0SD',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (3,'TCGA-A1-A0SE',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (4,'TCGA-A1-A0SF',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (5,'TCGA-A1-A0SG',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (6,'TCGA-A1-A0SH',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (7,'TCGA-A1-A0SI',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (8,'TCGA-A1-A0SJ',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (9,'TCGA-A1-A0SK',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (10,'TCGA-A1-A0SM',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (11,'TCGA-A1-A0SN',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (12,'TCGA-A1-A0SO',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (13,'TCGA-A1-A0SP',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (14,'TCGA-A1-A0SQ',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (15,'TCGA-XX-0800',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (16,'TCGA-XX-0900',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (17,'TCGA-AA-3664',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (18,'TCGA-AA-3665',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (19,'TCGA-A2-A04U',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (20,'TCGA-A1-A0SB',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (21,'TCGA-TEST-PATIENT-21',1); +INSERT INTO `patient` (`internal_id`, `stable_id`, `cancer_study_id`) VALUES (22,'TCGA-TEST-PATIENT-22',1); --- patient -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (1,'TCGA-A1-A0SB',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (2,'TCGA-A1-A0SD',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (3,'TCGA-A1-A0SE',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (4,'TCGA-A1-A0SF',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (5,'TCGA-A1-A0SG',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (6,'TCGA-A1-A0SH',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (7,'TCGA-A1-A0SI',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (8,'TCGA-A1-A0SJ',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (9,'TCGA-A1-A0SK',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (10,'TCGA-A1-A0SM',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (11,'TCGA-A1-A0SN',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (12,'TCGA-A1-A0SO',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (13,'TCGA-A1-A0SP',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (14,'TCGA-A1-A0SQ',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (15,'TCGA-XX-0800',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (16,'TCGA-XX-0900',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (17,'TCGA-AA-3664',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (18,'TCGA-AA-3665',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (19,'TCGA-A2-A04U',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (20,'TCGA-A1-A0SB',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (21,'TCGA-TEST-PATIENT-21',1); -INSERT INTO "patient" ("INTERNAL_ID","STABLE_ID","CANCER_STUDY_ID") VALUES (22,'TCGA-TEST-PATIENT-22',1); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (1,'TCGA-A1-A0SB-01','Primary Solid Tumor',1); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (2,'TCGA-A1-A0SD-01','Primary Solid Tumor',2); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (3,'TCGA-A1-A0SE-01','Primary Solid Tumor',3); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (4,'TCGA-A1-A0SF-01','Primary Solid Tumor',4); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (5,'TCGA-A1-A0SG-01','Primary Solid Tumor',5); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (6,'TCGA-A1-A0SH-01','Primary Solid Tumor',6); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (7,'TCGA-A1-A0SI-01','Primary Solid Tumor',7); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (8,'TCGA-A1-A0SJ-01','Primary Solid Tumor',8); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (9,'TCGA-A1-A0SK-01','Primary Solid Tumor',9); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (10,'TCGA-A1-A0SM-01','Primary Solid Tumor',10); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (11,'TCGA-A1-A0SN-01','Primary Solid Tumor',11); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (12,'TCGA-A1-A0SO-01','Primary Solid Tumor',12); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (13,'TCGA-A1-A0SP-01','Primary Solid Tumor',13); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (14,'TCGA-A1-A0SQ-01','Primary Solid Tumor',14); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (15,'TCGA-XX-0800-01','Primary Solid Tumor',15); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (16,'TCGA-XX-0900-01','Primary Solid Tumor',16); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (17,'TCGA-AA-3664-01','Primary Solid Tumor',17); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (18,'TCGA-AA-3665-01','Primary Solid Tumor',18); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (19,'TCGA-A1-A0SB-02','Primary Solid Tumor',1); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (20,'TCGA-A2-A04U-11','Solid Tissues Normal',19); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (21,'TCGA-A1-A0SB-11','Solid Tissues Normal',20); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (22,'TCGA-TEST-SAMPLE-22','Primary Solid Tumor',21); +INSERT INTO `sample` (`internal_id`, `stable_id`, `sample_type`, `patient_id`) VALUES (23,'TCGA-TEST-SAMPLE-23','Primary Solid Tumor',22); --- sample -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (1,'TCGA-A1-A0SB-01','Primary Solid Tumor',1); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (2,'TCGA-A1-A0SD-01','Primary Solid Tumor',2); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (3,'TCGA-A1-A0SE-01','Primary Solid Tumor',3); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (4,'TCGA-A1-A0SF-01','Primary Solid Tumor',4); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (5,'TCGA-A1-A0SG-01','Primary Solid Tumor',5); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (6,'TCGA-A1-A0SH-01','Primary Solid Tumor',6); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (7,'TCGA-A1-A0SI-01','Primary Solid Tumor',7); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (8,'TCGA-A1-A0SJ-01','Primary Solid Tumor',8); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (9,'TCGA-A1-A0SK-01','Primary Solid Tumor',9); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (10,'TCGA-A1-A0SM-01','Primary Solid Tumor',10); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (11,'TCGA-A1-A0SN-01','Primary Solid Tumor',11); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (12,'TCGA-A1-A0SO-01','Primary Solid Tumor',12); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (13,'TCGA-A1-A0SP-01','Primary Solid Tumor',13); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (14,'TCGA-A1-A0SQ-01','Primary Solid Tumor',14); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (15,'TCGA-XX-0800-01','Primary Solid Tumor',15); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (16,'TCGA-XX-0900-01','Primary Solid Tumor',16); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (17,'TCGA-AA-3664-01','Primary Solid Tumor',17); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (18,'TCGA-AA-3665-01','Primary Solid Tumor',18); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (19,'TCGA-A1-A0SB-02','Primary Solid Tumor',1); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (20,'TCGA-A2-A04U-11','Solid Tissues Normal',19); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (21,'TCGA-A1-A0SB-11','Solid Tissues Normal',20); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (22,'TCGA-TEST-SAMPLE-22','Primary Solid Tumor',21); -INSERT INTO "sample" ("INTERNAL_ID","STABLE_ID","SAMPLE_TYPE","PATIENT_ID") VALUES (23,'TCGA-TEST-SAMPLE-23','Primary Solid Tumor',22); +INSERT INTO `mutation_event` (`mutation_event_id`, `entrez_gene_id`, `chr`, `start_position`, `end_position`, `reference_allele`, `tumor_seq_allele`, `protein_change`, `mutation_type`, `ncbi_build`, `strand`, `variant_type`, `db_snp_rs`, `db_snp_val_status`, `refseq_mrna_id`, `codon_change`, `uniprot_accession`, `protein_pos_start`, `protein_pos_end`, `canonical_transcript`, `keyword`) VALUES (2038,672,'17',41244748,41244748,'G','A','Q934*','Nonsense_Mutation','37','+','SNP','rs80357223','unknown','NM_007294','c.(2800-2802)CAG>TAG','P38398',934,934,1,'BRCA1 truncating'); +INSERT INTO `mutation_event` (`mutation_event_id`, `entrez_gene_id`, `chr`, `start_position`, `end_position`, `reference_allele`, `tumor_seq_allele`, `protein_change`, `mutation_type`, `ncbi_build`, `strand`, `variant_type`, `db_snp_rs`, `db_snp_val_status`, `refseq_mrna_id`, `codon_change`, `uniprot_accession`, `protein_pos_start`, `protein_pos_end`, `canonical_transcript`, `keyword`) VALUES (22604,672,'17',41258504,41258504,'A','C','C61G','Missense_Mutation','37','+','SNP','rs28897672','byCluster','NM_007294','c.(181-183)TGT>GGT','P38398',61,61,1,'BRCA1 C61 missense'); +INSERT INTO `mutation_event` (`mutation_event_id`, `entrez_gene_id`, `chr`, `start_position`, `end_position`, `reference_allele`, `tumor_seq_allele`, `protein_change`, `mutation_type`, `ncbi_build`, `strand`, `variant_type`, `db_snp_rs`, `db_snp_val_status`, `refseq_mrna_id`, `codon_change`, `uniprot_accession`, `protein_pos_start`, `protein_pos_end`, `canonical_transcript`, `keyword`) VALUES (2039,672,'17',41276033,41276033,'C','T','C27_splice','Splice_Site','37','+','SNP','rs80358010','byCluster','NM_007294','c.e2+1','NA',-1,-1,1,'BRCA1 truncating'); --- mutation_event -INSERT INTO "mutation_event" ("MUTATION_EVENT_ID","ENTREZ_GENE_ID","CHR","START_POSITION","END_POSITION","REFERENCE_ALLELE","TUMOR_SEQ_ALLELE","PROTEIN_CHANGE","MUTATION_TYPE","NCBI_BUILD","STRAND","VARIANT_TYPE","DB_SNP_RS","DB_SNP_VAL_STATUS","REFSEQ_MRNA_ID","CODON_CHANGE","UNIPROT_ACCESSION","PROTEIN_POS_START","PROTEIN_POS_END","CANONICAL_TRANSCRIPT","KEYWORD") VALUES (2038,672,'17',41244748,41244748,'G','A','Q934*','Nonsense_Mutation','37','+','SNP','rs80357223','unknown','NM_007294','c.(2800-2802)CAG>TAG','P38398',934,934,1,'BRCA1 truncating'); -INSERT INTO "mutation_event" ("MUTATION_EVENT_ID","ENTREZ_GENE_ID","CHR","START_POSITION","END_POSITION","REFERENCE_ALLELE","TUMOR_SEQ_ALLELE","PROTEIN_CHANGE","MUTATION_TYPE","NCBI_BUILD","STRAND","VARIANT_TYPE","DB_SNP_RS","DB_SNP_VAL_STATUS","REFSEQ_MRNA_ID","CODON_CHANGE","UNIPROT_ACCESSION","PROTEIN_POS_START","PROTEIN_POS_END","CANONICAL_TRANSCRIPT","KEYWORD") VALUES (22604,672,'17',41258504,41258504,'A','C','C61G','Missense_Mutation','37','+','SNP','rs28897672','byCluster','NM_007294','c.(181-183)TGT>GGT','P38398',61,61,1,'BRCA1 C61 missense'); -INSERT INTO "mutation_event" ("MUTATION_EVENT_ID","ENTREZ_GENE_ID","CHR","START_POSITION","END_POSITION","REFERENCE_ALLELE","TUMOR_SEQ_ALLELE","PROTEIN_CHANGE","MUTATION_TYPE","NCBI_BUILD","STRAND","VARIANT_TYPE","DB_SNP_RS","DB_SNP_VAL_STATUS","REFSEQ_MRNA_ID","CODON_CHANGE","UNIPROT_ACCESSION","PROTEIN_POS_START","PROTEIN_POS_END","CANONICAL_TRANSCRIPT","KEYWORD") VALUES (2039,672,'17',41276033,41276033,'C','T','C27_splice','Splice_Site','37','+','SNP','rs80358010','byCluster','NM_007294','c.e2+1','NA',-1,-1,1,'BRCA1 truncating'); +INSERT INTO `mutation` (`mutation_event_id`, `genetic_profile_id`, `sample_id`, `entrez_gene_id`, `center`, `sequencer`, `mutation_status`, `validation_status`, `tumor_seq_allele1`, `tumor_seq_allele2`, `matched_norm_sample_barcode`, `match_norm_seq_allele1`, `match_norm_seq_allele2`, `tumor_validation_allele1`, `tumor_validation_allele2`, `match_norm_validation_allele1`, `match_norm_validation_allele2`, `verification_status`, `sequencing_phase`, `sequence_source`, `validation_method`, `score`, `bam_file`, `tumor_alt_count`, `tumor_ref_count`, `normal_alt_count`, `normal_ref_count`) VALUES (2038,6,6,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','G','A','TCGA-A1-A0SH-10A-03D-A099-09','G','A','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); +INSERT INTO `mutation` (`mutation_event_id`, `genetic_profile_id`, `sample_id`, `entrez_gene_id`, `center`, `sequencer`, `mutation_status`, `validation_status`, `tumor_seq_allele1`, `tumor_seq_allele2`, `matched_norm_sample_barcode`, `match_norm_seq_allele1`, `match_norm_seq_allele2`, `tumor_validation_allele1`, `tumor_validation_allele2`, `match_norm_validation_allele1`, `match_norm_validation_allele2`, `verification_status`, `sequencing_phase`, `sequence_source`, `validation_method`, `score`, `bam_file`, `tumor_alt_count`, `tumor_ref_count`, `normal_alt_count`, `normal_ref_count`) VALUES (22604,6,6,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','A','C','TCGA-A1-A0SH-10A-03D-A099-09','A','C','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); +INSERT INTO `mutation` (`mutation_event_id`, `genetic_profile_id`, `sample_id`, `entrez_gene_id`, `center`, `sequencer`, `mutation_status`, `validation_status`, `tumor_seq_allele1`, `tumor_seq_allele2`, `matched_norm_sample_barcode`, `match_norm_seq_allele1`, `match_norm_seq_allele2`, `tumor_validation_allele1`, `tumor_validation_allele2`, `match_norm_validation_allele1`, `match_norm_validation_allele2`, `verification_status`, `sequencing_phase`, `sequence_source`, `validation_method`, `score`, `bam_file`, `tumor_alt_count`, `tumor_ref_count`, `normal_alt_count`, `normal_ref_count`) VALUES (2039,6,12,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','T','T','TCGA-A1-A0SO-10A-03D-A099-09','T','T','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); --- mutation -INSERT INTO "mutation" ("MUTATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","ENTREZ_GENE_ID","CENTER","SEQUENCER","MUTATION_STATUS","VALIDATION_STATUS","TUMOR_SEQ_ALLELE1","TUMOR_SEQ_ALLELE2","MATCHED_NORM_SAMPLE_BARCODE","MATCH_NORM_SEQ_ALLELE1","MATCH_NORM_SEQ_ALLELE2","TUMOR_VALIDATION_ALLELE1","TUMOR_VALIDATION_ALLELE2","MATCH_NORM_VALIDATION_ALLELE1","MATCH_NORM_VALIDATION_ALLELE2","VERIFICATION_STATUS","SEQUENCING_PHASE","SEQUENCE_SOURCE","VALIDATION_METHOD","SCORE","BAM_FILE","TUMOR_ALT_COUNT","TUMOR_REF_COUNT","NORMAL_ALT_COUNT","NORMAL_REF_COUNT") VALUES (2038,6,6,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','G','A','TCGA-A1-A0SH-10A-03D-A099-09','G','A','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); -INSERT INTO "mutation" ("MUTATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","ENTREZ_GENE_ID","CENTER","SEQUENCER","MUTATION_STATUS","VALIDATION_STATUS","TUMOR_SEQ_ALLELE1","TUMOR_SEQ_ALLELE2","MATCHED_NORM_SAMPLE_BARCODE","MATCH_NORM_SEQ_ALLELE1","MATCH_NORM_SEQ_ALLELE2","TUMOR_VALIDATION_ALLELE1","TUMOR_VALIDATION_ALLELE2","MATCH_NORM_VALIDATION_ALLELE1","MATCH_NORM_VALIDATION_ALLELE2","VERIFICATION_STATUS","SEQUENCING_PHASE","SEQUENCE_SOURCE","VALIDATION_METHOD","SCORE","BAM_FILE","TUMOR_ALT_COUNT","TUMOR_REF_COUNT","NORMAL_ALT_COUNT","NORMAL_REF_COUNT") VALUES (22604,6,6,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','A','C','TCGA-A1-A0SH-10A-03D-A099-09','A','C','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); -INSERT INTO "mutation" ("MUTATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","ENTREZ_GENE_ID","CENTER","SEQUENCER","MUTATION_STATUS","VALIDATION_STATUS","TUMOR_SEQ_ALLELE1","TUMOR_SEQ_ALLELE2","MATCHED_NORM_SAMPLE_BARCODE","MATCH_NORM_SEQ_ALLELE1","MATCH_NORM_SEQ_ALLELE2","TUMOR_VALIDATION_ALLELE1","TUMOR_VALIDATION_ALLELE2","MATCH_NORM_VALIDATION_ALLELE1","MATCH_NORM_VALIDATION_ALLELE2","VERIFICATION_STATUS","SEQUENCING_PHASE","SEQUENCE_SOURCE","VALIDATION_METHOD","SCORE","BAM_FILE","TUMOR_ALT_COUNT","TUMOR_REF_COUNT","NORMAL_ALT_COUNT","NORMAL_REF_COUNT") VALUES (2039,6,12,672,'genome.wustl.edu','IlluminaGAIIx','Germline','Unknown','T','T','TCGA-A1-A0SO-10A-03D-A099-09','T','T','NA','NA','NA','NA','Unknown','Phase_IV','Capture','NA','1','dbGAP',-1,-1,-1,-1); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (1,'study_tcga_pub_all','other',1,'All Tumors','All tumor samples (14 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (2,'study_tcga_pub_acgh','other',1,'Tumors aCGH','All tumors with aCGH data (778 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (3,'study_tcga_pub_cnaseq','other',1,'Tumors with sequencing and aCGH data','All tumor samples that have CNA and sequencing data (482 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (4,'study_tcga_pub_complete','other',1,'Complete samples (mutations,copy-number,expression)','Samples with complete data (463 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (5,'study_tcga_pub_log2CNA','other',1,'Tumors log2 copy-number','All tumors with log2 copy-number data (778 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (6,'study_tcga_pub_methylation_hm27','other',1,'Tumors with methylation data','All samples with methylation (HM27) data (311 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (7,'study_tcga_pub_mrna','other',1,'Tumors with mRNA data (Agilent microarray)','All samples with mRNA expression data (526 samples)'); +INSERT INTO `sample_list` (`list_id`, `stable_id`, `category`, `cancer_study_id`, `name`, `description`) VALUES (8,'study_tcga_pub_sequenced','other',1,'Sequenced Tumors','All sequenced samples (507 samples)'); --- sample_list -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (1,'study_tcga_pub_all','other',1,'All Tumors','All tumor samples (14 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (2,'study_tcga_pub_acgh','other',1,'Tumors aCGH','All tumors with aCGH data (778 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (3,'study_tcga_pub_cnaseq','other',1,'Tumors with sequencing and aCGH data','All tumor samples that have CNA and sequencing data (482 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (4,'study_tcga_pub_complete','other',1,'Complete samples (mutations,copy-number,expression)','Samples with complete data (463 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (5,'study_tcga_pub_log2CNA','other',1,'Tumors log2 copy-number','All tumors with log2 copy-number data (778 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (6,'study_tcga_pub_methylation_hm27','other',1,'Tumors with methylation data','All samples with methylation (HM27) data (311 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (7,'study_tcga_pub_mrna','other',1,'Tumors with mRNA data (Agilent microarray)','All samples with mRNA expression data (526 samples)'); -INSERT INTO "sample_list" ("LIST_ID","STABLE_ID","CATEGORY","CANCER_STUDY_ID","NAME","DESCRIPTION") VALUES (8,'study_tcga_pub_sequenced','other',1,'Sequenced Tumors','All sequenced samples (507 samples)'); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,1); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,4); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,5); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,7); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,11); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,13); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (1,14); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,1); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,4); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,5); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,7); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,11); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,13); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (2,14); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (3,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (4,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,1); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,4); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,5); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,7); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,11); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,13); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (5,14); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (6,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,12); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (7,13); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,2); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,3); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,6); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,8); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,9); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,10); +INSERT INTO `sample_list_list` (`list_id`, `sample_id`) VALUES (8,12); --- sample_list_list -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,1); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,4); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,5); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,7); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,11); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,13); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (1,14); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,1); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,4); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,5); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,7); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,11); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,13); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (2,14); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (3,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (4,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,1); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,4); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,5); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,7); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,11); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,13); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (5,14); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (6,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,12); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (7,13); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,2); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,3); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,6); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,8); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,9); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,10); -INSERT INTO "sample_list_list" ("LIST_ID","SAMPLE_ID") VALUES (8,12); +INSERT INTO `sample_cna_event` (`cna_event_id`, `sample_id`, `genetic_profile_id`) VALUES (2774,4,2); +INSERT INTO `sample_cna_event` (`cna_event_id`, `sample_id`, `genetic_profile_id`) VALUES (17280,12,2); +INSERT INTO `sample_cna_event` (`cna_event_id`, `sample_id`, `genetic_profile_id`) VALUES (26161,12,2); +INSERT INTO `sample_cna_event` (`cna_event_id`, `sample_id`, `genetic_profile_id`) VALUES (17280,13,2); +INSERT INTO `sample_cna_event` (`cna_event_id`, `sample_id`, `genetic_profile_id`) VALUES (26161,14,2); --- sample_cna_event -INSERT INTO "sample_cna_event" ("CNA_EVENT_ID","SAMPLE_ID","GENETIC_PROFILE_ID") VALUES (2774,4,2); -INSERT INTO "sample_cna_event" ("CNA_EVENT_ID","SAMPLE_ID","GENETIC_PROFILE_ID") VALUES (17280,12,2); -INSERT INTO "sample_cna_event" ("CNA_EVENT_ID","SAMPLE_ID","GENETIC_PROFILE_ID") VALUES (26161,12,2); -INSERT INTO "sample_cna_event" ("CNA_EVENT_ID","SAMPLE_ID","GENETIC_PROFILE_ID") VALUES (17280,13,2); -INSERT INTO "sample_cna_event" ("CNA_EVENT_ID","SAMPLE_ID","GENETIC_PROFILE_ID") VALUES (26161,14,2); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (2774,2,4,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (17280,2,12,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (26161,2,12,'Putative_Passenger','Pathogenic','Tier 2','Potentially Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (17280,2,13,'Putative_Passenger','Pathogenic','Tier 1','Highly Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (26161,2,14,'Putative_Driver','Pathogenic','Tier 2','Potentially Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (26161,6,6,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (22604,6,6,'Putative_Passenger','Pathogenic','Tier 2','Potentially Actionable'); +INSERT INTO `alteration_driver_annotation` (`alteration_event_id`, `genetic_profile_id`, `sample_id`, `driver_filter`, `driver_filter_annotation`, `driver_tiers_filter`, `driver_tiers_filter_annotation`) VALUES (2039,6,12,'Putative_Passenger','Pathogenic','Tier 1','Highly Actionable'); --- alteration_driver_annotation - -- from sample_cna_event -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (2774,2,4,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (17280,2,12,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (26161,2,12,'Putative_Passenger','Pathogenic','Tier 2','Potentially Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (17280,2,13,'Putative_Passenger','Pathogenic','Tier 1','Highly Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (26161,2,14,'Putative_Driver','Pathogenic','Tier 2','Potentially Actionable'); - -- from mutation -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (26161,6,6,'Putative_Driver','Pathogenic','Tier 1','Highly Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (22604,6,6,'Putative_Passenger','Pathogenic','Tier 2','Potentially Actionable'); -INSERT INTO "alteration_driver_annotation" ("ALTERATION_EVENT_ID","GENETIC_PROFILE_ID","SAMPLE_ID","DRIVER_FILTER","DRIVER_FILTER_ANNOTATION","DRIVER_TIERS_FILTER","DRIVER_TIERS_FILTER_ANNOTATION") VALUES (2039,6,12,'Putative_Passenger','Pathogenic','Tier 1','Highly Actionable'); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (1,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (1,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (2,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (2,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (2,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (2,5,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (2,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (3,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (3,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (3,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (3,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (4,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (4,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (5,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (5,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (6,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (6,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (6,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (6,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (7,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (7,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (8,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (8,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (8,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (8,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (9,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (9,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (9,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (9,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (10,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (10,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (10,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (10,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (11,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (11,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (12,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (12,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (12,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (12,6,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (13,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (13,3,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (13,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (14,2,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (14,4,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (22,11,NULL); +INSERT INTO `sample_profile` (`sample_id`, `genetic_profile_id`, `panel_id`) VALUES (23,12,NULL); --- sample_profile -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (1,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (1,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (2,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (2,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (2,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (2,5,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (2,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (3,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (3,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (3,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (3,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (4,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (4,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (5,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (5,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (6,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (6,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (6,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (6,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (7,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (7,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (8,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (8,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (8,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (8,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (9,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (9,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (9,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (9,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (10,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (10,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (10,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (10,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (11,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (11,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (12,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (12,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (12,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (12,6,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (13,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (13,3,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (13,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (14,2,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (14,4,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (22,11,NULL); -INSERT INTO "sample_profile" ("SAMPLE_ID","GENETIC_PROFILE_ID","PANEL_ID") VALUES (23,12,NULL); +INSERT INTO users (`email`, `name`, `enabled`) values ('jami@gmail.com','Jami Bax',1); +INSERT INTO users (`email`, `name`, `enabled`) values ('Lonnie@openid.org','Lonnie Penaloza',0); +INSERT INTO users (`email`, `name`, `enabled`) values ('Dhorak@yahoo.com','Darryl Horak',1); --- users -INSERT INTO users (EMAIL,NAME,ENABLED) values ('jami@gmail.com','Jami Bax',1); -INSERT INTO users (EMAIL,NAME,ENABLED) values ('Lonnie@openid.org','Lonnie Penaloza',0); -INSERT INTO users (EMAIL,NAME,ENABLED) values ('Dhorak@yahoo.com','Darryl Horak',1); +INSERT INTO authorities (`email`, `authority`) values ('jami@gmail.com','ROLE_USER'); +INSERT INTO authorities (`email`, `authority`) values ('Lonnie@openid.org','ROLE_USER'); +INSERT INTO authorities (`email`, `authority`) values ('Dhorak@yahoo.com','ROLE_USER'); +INSERT INTO authorities (`email`, `authority`) values ('Dhorak@yahoo.com','ROLE_MANAGER'); --- authorities -INSERT INTO authorities (EMAIL,AUTHORITY) values ('jami@gmail.com','ROLE_USER'); -INSERT INTO authorities (EMAIL,AUTHORITY) values ('Lonnie@openid.org','ROLE_USER'); -INSERT INTO authorities (EMAIL,AUTHORITY) values ('Dhorak@yahoo.com','ROLE_USER'); -INSERT INTO authorities (EMAIL,AUTHORITY) values ('Dhorak@yahoo.com','ROLE_MANAGER'); +INSERT INTO `info` (`db_schema_version`, `geneset_version`, `derived_table_schema_version`) +VALUES ('2.14.5', 'test_geneset_version', '1.0.6'); --- info -UPDATE info SET GENESET_VERSION = 'test_geneset_version'; +--geneset +INSERT INTO geneset (`id`, `genetic_entity_id`, `external_id`, `name`, `description`, `ref_link`) VALUES (1, 1, 'HGNC:1100', 'BRCA1', 'Breast cancer type 1 susceptibility protein', 'https://www.ncbi.nlm.nih.gov/gene/672'); +INSERT INTO geneset (`id`, `genetic_entity_id`, `external_id`, `name`, `description`, `ref_link`) VALUES (2, 2, 'HGNC:1101', 'BRCA2', 'Breast cancer type 2 susceptibility protein', 'https://www.ncbi.nlm.nih.gov/gene/675'); --- geneset -INSERT INTO geneset (ID, GENETIC_ENTITY_ID, EXTERNAL_ID, NAME, DESCRIPTION, REF_LINK) VALUES ('1', '1', 'HGNC:1100', 'BRCA1', 'Breast cancer type 1 susceptibility protein', 'https://www.ncbi.nlm.nih.gov/gene/672'); -INSERT INTO geneset (ID, GENETIC_ENTITY_ID, EXTERNAL_ID, NAME, DESCRIPTION, REF_LINK) VALUES ('2', '2', 'HGNC:1101', 'BRCA2', 'Breast cancer type 2 susceptibility protein', 'https://www.ncbi.nlm.nih.gov/gene/675');