perf(import): bulk-load tab-delim matrix + genetic_entity inserts

src/main/java/org/mskcc/cbio/portal/dao/DaoGeneticEntity.java

@@ -30,18 +30,27 @@ private enum SqlAction {
 
     public static GeneticEntity addNewGeneticEntity(GeneticEntity geneticEntity) throws DaoException {
 
+        long entityId = ClickHouseAutoIncrement.nextId("seq_genetic_entity");
+        geneticEntity.setId((int) entityId);
+
+        if (ClickHouseBulkLoader.isBulkLoad()) {
+            ClickHouseBulkLoader.getClickHouseBulkLoader("genetic_entity").insertRecord(
+                Long.toString(entityId),
+                geneticEntity.getEntityType(),
+                geneticEntity.getStableId());
+            return geneticEntity;
+        }
+
         Connection con = null;
         PreparedStatement pstmt = null;
         try {
             con = JdbcUtil.getDbConnection(DaoGeneticEntity.class);
-            long entityId = ClickHouseAutoIncrement.nextId("seq_genetic_entity");
             pstmt = con.prepareStatement("INSERT INTO genetic_entity (`id`, `entity_type`, `stable_id`) "
                     + "VALUES(?,?,?)");
             pstmt.setLong(1, entityId);
             pstmt.setString(2, geneticEntity.getEntityType());
             pstmt.setString(3, geneticEntity.getStableId());
             pstmt.executeUpdate();
-            geneticEntity.setId((int) entityId);
         } catch (SQLException e) {
             throw new DaoException(e);
         } finally {

src/main/java/org/mskcc/cbio/portal/scripts/ImportGenericAssayEntity.java

@@ -41,6 +41,7 @@
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import org.mskcc.cbio.portal.dao.ClickHouseBulkLoader;
 import org.mskcc.cbio.portal.dao.DaoGenericAssay;
 import org.mskcc.cbio.portal.dao.DaoGeneticEntity;
 import org.mskcc.cbio.portal.model.shared.EntityType;
@@ -247,7 +248,18 @@ public static void importData(File dataFile, GeneticAlterationType geneticAltera
         }
 
         reader.close();
-
+
+        // Flush any buffered genetic_entity inserts so that a subsequent SELECT
+        // (e.g. GenericAssayMetaUtils.buildGenericAssayStableIdToEntityIdMap)
+        // in the same JVM sees them. ImportProfileData runs this method
+        // immediately before ImportTabDelimData for GENERIC_ASSAY profiles.
+        // bulkLoadOff() restores the global flag — ImportTabDelimData will
+        // turn it back on for the matrix import.
+        if (ClickHouseBulkLoader.isBulkLoad()) {
+            ClickHouseBulkLoader.flushAll();
+            ClickHouseBulkLoader.bulkLoadOff();
+        }
+
         ProgressMonitor.setCurrentMessage("Finished loading generic assay.\n");
     }
 

src/main/java/org/mskcc/cbio/portal/scripts/ImportTabDelimData.java

@@ -280,11 +280,15 @@ void importDataInternal() throws Exception {
 
             geneticAlterationImporter.initialize();
 
+            // Bulk-load matrix data via ClickHouseBulkLoader's
+            // INSERT ... FORMAT TSVWithNames path — one round-trip per profile
+            // instead of one JDBC INSERT per gene.
+            ClickHouseBulkLoader.bulkLoadOn();
+
             //cache for data found in  cna_event' table:
             Set<CnaEvent.Event> existingCnaEvents = new HashSet<>();
             if (isDiscretizedCnaProfile) {
                 existingCnaEvents.addAll(DaoCnaEvent.getAllCnaEvents());
-                ClickHouseBulkLoader.bulkLoadOn();
             }
 
             // load entities map from database
@@ -368,6 +372,7 @@ void importDataInternal() throws Exception {
             DaoSampleProfile.upsertSampleToProfileMapping(orderedSampleList, geneticProfileId, genePanelId);
             if (ClickHouseBulkLoader.isBulkLoad()) {
                 ClickHouseBulkLoader.flushAll();
+                ClickHouseBulkLoader.bulkLoadOff();
             }
             geneticAlterationImporter.complete();