Use newer dl2 file for model training tests

src/ctapipe/conftest.py

@@ -185,6 +185,13 @@ def prod5_gamma_lapalma_simtel_path():
     )
 
 
+@pytest.fixture(scope="session")
+def prod6_gamma_lapalma_simtel_path():
+    return get_dataset_path(
+        "gamma_20deg_180deg_run2___cta-prod6-2156m-LaPalma-dark_cone2.simtel.zst"
+    )
+
+
 @pytest.fixture(scope="session")
 def prod5_proton_simtel_path():
     return get_dataset_path(
@@ -194,7 +201,12 @@ def prod5_proton_simtel_path():
 
 @pytest.fixture(scope="session")
 def proton_dl2_train_small_h5():
-    return get_dataset_path("proton_dl2_train_small.dl2.h5")
+    return get_dataset_path("proton_train_0.23.1.dl2.h5")
+
+
+@pytest.fixture(scope="session")
+def gamma_dl2_train_small_h5():
+    return get_dataset_path("gamma_diffuse_train_0.23.1.dl2.h5")
 
 
 @pytest.fixture(scope="session")
@@ -604,7 +616,7 @@ def model_tmp_path(tmp_path_factory):
 
 
 @pytest.fixture(scope="session")
-def energy_regressor_path(model_tmp_path):
+def energy_regressor_path(model_tmp_path, gamma_dl2_train_small_h5):
     from ctapipe.tools.train_energy_regressor import TrainEnergyRegressor
 
     out_file = model_tmp_path / "energy.pkl"
@@ -618,7 +630,7 @@ def energy_regressor_path(model_tmp_path):
         ret = run_tool(
             tool,
             argv=[
-                "--input=dataset://gamma_diffuse_dl2_train_small.dl2.h5",
+                f"--input={gamma_dl2_train_small_h5}",
                 f"--output={out_file}",
                 f"--config={config}",
                 "--log-level=INFO",
@@ -630,15 +642,14 @@ def energy_regressor_path(model_tmp_path):
 
 
 @pytest.fixture(scope="session")
-def gamma_train_clf(model_tmp_path, energy_regressor_path):
+def gamma_train_clf(model_tmp_path, energy_regressor_path, gamma_dl2_train_small_h5):
     from ctapipe.tools.apply_models import ApplyModels
 
-    inpath = "dataset://gamma_diffuse_dl2_train_small.dl2.h5"
     outpath = model_tmp_path / "gamma_train_clf.dl2.h5"
     run_tool(
         ApplyModels(),
         argv=[
-            f"--input={inpath}",
+            f"--input={gamma_dl2_train_small_h5}",
             f"--output={outpath}",
             f"--reconstructor={energy_regressor_path}",
         ],
@@ -648,15 +659,14 @@ def gamma_train_clf(model_tmp_path, energy_regressor_path):
 
 
 @pytest.fixture(scope="session")
-def proton_train_clf(model_tmp_path, energy_regressor_path):
+def proton_train_clf(model_tmp_path, energy_regressor_path, proton_dl2_train_small_h5):
     from ctapipe.tools.apply_models import ApplyModels
 
-    inpath = "dataset://proton_dl2_train_small.dl2.h5"
     outpath = model_tmp_path / "proton_train_clf.dl2.h5"
     run_tool(
         ApplyModels(),
         argv=[
-            f"--input={inpath}",
+            f"--input={proton_dl2_train_small_h5}",
             f"--output={outpath}",
             f"--reconstructor={energy_regressor_path}",
         ],

src/ctapipe/io/tests/test_merge.py

@@ -6,7 +6,6 @@
 from ctapipe.instrument.subarray import SubarrayDescription
 from ctapipe.io.astropy_helpers import read_table
 from ctapipe.io.tests.test_astropy_helpers import assert_table_equal
-from ctapipe.utils.datasets import get_dataset_path
 
 
 def compare_table(in1, in2, merged, table):
@@ -113,11 +112,9 @@ def test_simple(tmp_path, gamma_train_clf, proton_train_clf):
             compare_stats_table(in1, in2, merged, table)
 
 
-def test_append(tmp_path, gamma_train_clf, proton_train_clf):
+def test_append(tmp_path, gamma_train_clf, proton_train_clf, gamma_dl2_train_small_h5):
     from ctapipe.io.hdf5merger import CannotMerge, HDF5Merger
 
-    gamma_train_en = get_dataset_path("gamma_diffuse_dl2_train_small.dl2.h5")
-
     output = tmp_path / "merged_simple.dl2.h5"
     shutil.copy2(gamma_train_clf, output)
 
@@ -129,7 +126,7 @@ def test_append(tmp_path, gamma_train_clf, proton_train_clf):
         with pytest.raises(
             CannotMerge, match="Required node .*/energy/ExtraTreesRegressor"
         ):
-            merger(gamma_train_en)
+            merger(gamma_dl2_train_small_h5)
 
 
 def test_filter_column(tmp_path, dl2_shower_geometry_file):

src/ctapipe/tools/tests/test_apply_models.py

@@ -22,7 +22,6 @@
     DL2_TEL_PARTICLETYPE_GROUP,
 )
 from ctapipe.io.tests.test_table_loader import check_equal_array_event_order
-from ctapipe.utils.datasets import get_dataset_path
 
 
 def test_apply_energy_regressor(
@@ -116,13 +115,14 @@ def test_apply_all(
     energy_regressor_path,
     particle_classifier_path,
     disp_reconstructor_path,
+    gamma_dl2_train_small_h5,
     tmp_path,
 ):
     from ctapipe.tools.apply_models import ApplyModels
 
     disp_reconstructor_path, _ = disp_reconstructor_path
 
-    input_path = get_dataset_path("gamma_diffuse_dl2_train_small.dl2.h5")
+    input_path = gamma_dl2_train_small_h5
     output_path = tmp_path / "particle-and-energy-and-disp.dl2.h5"
 
     ret = run_tool(

src/ctapipe/tools/tests/test_process_ml.py

@@ -14,20 +14,22 @@
     DL2_TEL_GROUP,
 )
 
+ALLOWED_TELS_PROD6_NORTH_ALPHA = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 19]
+
 
 def test_process_apply_energy(
-    tmp_path, energy_regressor_path, prod5_gamma_lapalma_simtel_path
+    tmp_path, energy_regressor_path, prod6_gamma_lapalma_simtel_path
 ):
     from ctapipe.tools.process import ProcessorTool
 
     output = tmp_path / "gamma_prod5.dl2_energy.h5"
 
     config_path = tmp_path / "config.json"
 
-    input_url = prod5_gamma_lapalma_simtel_path
+    input_url = prod6_gamma_lapalma_simtel_path
 
     # la palma alpha config
-    allowed_tels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 19, 35]
+    allowed_tels = ALLOWED_TELS_PROD6_NORTH_ALPHA
     config = {
         "ProcessorTool": {
             "EventSource": {
@@ -69,17 +71,17 @@ def test_process_apply_classification(
     tmp_path,
     energy_regressor_path,
     particle_classifier_path,
-    prod5_gamma_lapalma_simtel_path,
+    prod6_gamma_lapalma_simtel_path,
 ):
     from ctapipe.tools.process import ProcessorTool
 
     output = tmp_path / "gamma_prod5.dl2_energy.h5"
 
     config_path = tmp_path / "config.json"
 
-    input_url = prod5_gamma_lapalma_simtel_path
+    input_url = prod6_gamma_lapalma_simtel_path
 
-    allowed_tels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 19, 35]
+    allowed_tels = ALLOWED_TELS_PROD6_NORTH_ALPHA
     config = {
         "ProcessorTool": {
             "EventSource": {
@@ -127,7 +129,7 @@ def test_process_apply_disp(
     tmp_path,
     energy_regressor_path,
     disp_reconstructor_path,
-    prod5_gamma_lapalma_simtel_path,
+    prod6_gamma_lapalma_simtel_path,
 ):
     from ctapipe.tools.process import ProcessorTool
 
@@ -137,9 +139,9 @@ def test_process_apply_disp(
 
     config_path = tmp_path / "config.json"
 
-    input_url = prod5_gamma_lapalma_simtel_path
+    input_url = prod6_gamma_lapalma_simtel_path
 
-    allowed_tels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 19, 35]
+    allowed_tels = ALLOWED_TELS_PROD6_NORTH_ALPHA
     config = {
         "ProcessorTool": {
             "EventSource": {

src/ctapipe/tools/tests/test_train.py

@@ -4,6 +4,7 @@
 from ctapipe.core import ToolConfigurationError, run_tool
 from ctapipe.exceptions import TooFewEvents
 from ctapipe.io import read_table
+from ctapipe.utils import get_dataset_path
 from ctapipe.utils.datasets import resource_file
 
 
@@ -26,7 +27,7 @@ def test_train_disp_reconstructor(disp_reconstructor_path):
 
     DispReconstructor.read(model_path)
 
-    cv_table = read_table(cv_path, "/cv_predictions/LST_LST_LSTCam")
+    cv_table = read_table(cv_path, "/cv_predictions/LST_LST_LSTcam")
     disp = cv_table["disp_parameter"]
     true_disp = cv_table["truth"]
     accuracy = np.count_nonzero(np.sign(disp) == np.sign(true_disp)) / len(disp)
@@ -53,7 +54,7 @@ def test_too_few_events(tmp_path, dl2_shower_geometry_file):
         )
 
 
-def test_sampling(tmp_path, dl2_shower_geometry_file):
+def test_sampling(tmp_path, gamma_dl2_train_small_h5):
     from ctapipe.tools.train_energy_regressor import TrainEnergyRegressor
 
     tool = TrainEnergyRegressor()
@@ -63,7 +64,7 @@ def test_sampling(tmp_path, dl2_shower_geometry_file):
     run_tool(
         tool,
         argv=[
-            "--input=dataset://gamma_diffuse_dl2_train_small.dl2.h5",
+            f"--input={gamma_dl2_train_small_h5}",
             f"--output={out_file}",
             f"--config={config}",
             "--log-level=INFO",
@@ -232,3 +233,98 @@ def test_no_cross_validation(tmp_path):
         ],
     )
     assert ret == 0
+
+
+def test_train_models_0_17(tmp_path):
+    """Test for training disp reconstructor on the older (ctapipe 0.17) test data"""
+    from ctapipe.reco.sklearn import (
+        DispReconstructor,
+        EnergyRegressor,
+        ParticleClassifier,
+    )
+    from ctapipe.tools.apply_models import ApplyModels
+    from ctapipe.tools.train_disp_reconstructor import TrainDispReconstructor
+    from ctapipe.tools.train_energy_regressor import TrainEnergyRegressor
+    from ctapipe.tools.train_particle_classifier import TrainParticleClassifier
+
+    gamma_input = get_dataset_path("gamma_diffuse_dl2_train_small.dl2.h5")
+    proton_input = get_dataset_path("proton_dl2_train_small.dl2.h5")
+
+    # Train energy model first, to have the energy usable in later models
+    energy_tool = TrainEnergyRegressor()
+    energy_config = resource_file("train_energy_regressor.yaml")
+    energy_model = tmp_path / "energy_regressor.pkl"
+    ret = run_tool(
+        energy_tool,
+        argv=[
+            f"--input={gamma_input}",
+            f"--output={energy_model}",
+            f"--config={energy_config}",
+            "--log-level=INFO",
+        ],
+    )
+    assert ret == 0
+    assert energy_model.is_file()
+    EnergyRegressor.read(energy_model)
+
+    # apply to proton and gamma for training of disp and particle
+    gamma_train_clf = tmp_path / "gamma_train.dl2.h5"
+    proton_train_clf = tmp_path / "proton_train.dl2.h5"
+
+    apply_models = ApplyModels()
+    for infile, outfile in zip(
+        [gamma_input, proton_input], [gamma_train_clf, proton_train_clf]
+    ):
+        ret = run_tool(
+            apply_models,
+            argv=[
+                f"--input={infile}",
+                f"--output={outfile}",
+                f"--reconstructor={energy_model}",
+            ],
+        )
+        assert ret == 0
+
+    disp_tool = TrainDispReconstructor()
+    disp_config = resource_file("train_disp_reconstructor.yaml")
+
+    disp_model = tmp_path / "disp_reconstructor.pkl"
+    disp_cv_out_file = tmp_path / "disp_cv_results.h5"
+
+    ret = run_tool(
+        disp_tool,
+        argv=[
+            f"--input={gamma_train_clf}",
+            f"--output={disp_model}",
+            f"--config={disp_config}",
+            f"--cv-output={disp_cv_out_file}",
+            "--log-level=INFO",
+        ],
+    )
+    assert ret == 0
+    assert disp_cv_out_file.exists()
+
+    DispReconstructor.read(disp_model)
+
+    cv_table = read_table(disp_cv_out_file, "/cv_predictions/LST_LST_LSTCam")
+    disp = cv_table["disp_parameter"]
+    true_disp = cv_table["truth"]
+    accuracy = np.count_nonzero(np.sign(disp) == np.sign(true_disp)) / len(disp)
+    assert accuracy > 0.75
+
+    classifier_tool = TrainParticleClassifier()
+    classifier_config = resource_file("train_particle_classifier.yaml")
+    classifier_model = tmp_path / "particle_classifier.pkl"
+
+    ret = run_tool(
+        classifier_tool,
+        argv=[
+            f"--signal={gamma_train_clf}",
+            f"--background={proton_train_clf}",
+            f"--output={classifier_model}",
+            f"--config={classifier_config}",
+            "--log-level=INFO",
+        ],
+    )
+    assert ret == 0
+    ParticleClassifier.read(classifier_model)