diff --git a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java index 8f2ed303..b0ae2d33 100644 --- a/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java +++ b/src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java @@ -30,18 +30,27 @@ private enum SqlAction { public static GeneticEntity addNewGeneticEntity(GeneticEntity geneticEntity) throws DaoException { + long entityId = ClickHouseAutoIncrement.nextId("seq_genetic_entity"); + geneticEntity.setId((int) entityId); + + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.getClickHouseBulkLoader("genetic_entity").insertRecord( + Long.toString(entityId), + geneticEntity.getEntityType(), + geneticEntity.getStableId()); + return geneticEntity; + } + Connection con = null; PreparedStatement pstmt = null; try { con = JdbcUtil.getDbConnection(DaoGeneticEntity.class); - long entityId = ClickHouseAutoIncrement.nextId("seq_genetic_entity"); pstmt = con.prepareStatement("INSERT INTO genetic_entity (`id`, `entity_type`, `stable_id`) " + "VALUES(?,?,?)"); pstmt.setLong(1, entityId); pstmt.setString(2, geneticEntity.getEntityType()); pstmt.setString(3, geneticEntity.getStableId()); pstmt.executeUpdate(); - geneticEntity.setId((int) entityId); } catch (SQLException e) { throw new DaoException(e); } finally { diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java index 0c629364..e55bc716 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java @@ -41,6 +41,7 @@ import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; +import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader; import org.mskcc.cbio.portal.dao.DaoGenericAssay; import org.mskcc.cbio.portal.dao.DaoGeneticEntity; import org.mskcc.cbio.portal.model.shared.EntityType; @@ -247,7 +248,18 @@ public static void importData(File dataFile, GeneticAlterationType geneticAltera } reader.close(); - + + // Flush any buffered genetic_entity inserts so that a subsequent SELECT + // (e.g. GenericAssayMetaUtils.buildGenericAssayStableIdToEntityIdMap) + // in the same JVM sees them. ImportProfileData runs this method + // immediately before ImportTabDelimData for GENERIC_ASSAY profiles. + // bulkLoadOff() restores the global flag — ImportTabDelimData will + // turn it back on for the matrix import. + if (ClickHouseBulkLoader.isBulkLoad()) { + ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.bulkLoadOff(); + } + ProgressMonitor.setCurrentMessage("Finished loading generic assay.\n"); } diff --git a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java index ff346e77..b51023ea 100644 --- a/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java +++ b/src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java @@ -280,11 +280,15 @@ void importDataInternal() throws Exception { geneticAlterationImporter.initialize(); + // Bulk-load matrix data via ClickHouseBulkLoader's + // INSERT ... FORMAT TSVWithNames path — one round-trip per profile + // instead of one JDBC INSERT per gene. + ClickHouseBulkLoader.bulkLoadOn(); + //cache for data found in cna_event' table: Set existingCnaEvents = new HashSet<>(); if (isDiscretizedCnaProfile) { existingCnaEvents.addAll(DaoCnaEvent.getAllCnaEvents()); - ClickHouseBulkLoader.bulkLoadOn(); } // load entities map from database @@ -368,6 +372,7 @@ void importDataInternal() throws Exception { DaoSampleProfile.upsertSampleToProfileMapping(orderedSampleList, geneticProfileId, genePanelId); if (ClickHouseBulkLoader.isBulkLoad()) { ClickHouseBulkLoader.flushAll(); + ClickHouseBulkLoader.bulkLoadOff(); } geneticAlterationImporter.complete();