Add monthly daly reporting to individual history tracker

src/tlo/analysis/utils.py

@@ -413,14 +413,14 @@ def reconstruct_individual_histories(df):
     # Collapse into 'entity', 'date', 'event_name', 'Info' format where 'Info' is dict listing attributes
     # (e.g. {a1:v1, a2:v2, a3:v3, ...} )
     df_collapsed = (
-            df.groupby(['entity', 'date', 'event_name'], sort=False)
+            df.groupby(['entity', 'date', 'event_name', 'event_tag'], sort=False)
               .apply(lambda g: dict(zip(g['attribute'], g['value'])))
               .reset_index(name='Info')
         )
 
     df_final = (
         df_collapsed
-            .sort_values(by=['entity', 'date'])
+            .sort_values(by=['entity', 'date', 'event_tag'])
             .reset_index(drop=True)
     )
 
@@ -430,8 +430,6 @@ def reconstruct_individual_histories(df):
     if len(problems)>0:
         print("Values didn't change but were still detected")
         print(problems)
-
-
 
     return df_final
 
@@ -485,8 +483,6 @@ def extract_individual_histories(results_folder: Path,
         # Combine all dfs into a single DataFrame
         res[draw] = pd.concat(dfs_from_runs, ignore_index=True)
 
-        res[0].to_csv('individual_histories.csv')
-
     return res
 
 

src/tlo/methods/healthburden.py

@@ -19,6 +19,7 @@
     get_gbd_causes_not_represented_in_disease_modules,
 )
 from tlo.methods.demography import age_at_date
+from tlo.notify import notifier
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -614,6 +615,20 @@ def apply(self, population):
         # Multiply 1/12 as these weights are for one month only
         disease_specific_daly_values_this_month = disease_specific_daly_values_this_month * (1 / 12)
 
+        if notifier.has_listeners("healthburden.monthly_daly_report"):
+            # Do not dispatch individuals or causes that have zero dalys reported this month
+            monthly_dalys = disease_specific_daly_values_this_month
+            monthly_dalys_nonzero = (
+                monthly_dalys.loc[(monthly_dalys != 0).any(axis=1), (monthly_dalys != 0).any(axis=0)])
+
+            # Only retain non-zero info
+            data = {
+                idx: {col: val for col, val in row.items() if val > 0}
+                for idx, row in monthly_dalys_nonzero.iterrows()
+            }
+
+            notifier.dispatch("healthburden.monthly_daly_report", data=data)
+
         # 4) Summarise the results for this month wrt sex/age/wealth
         # - merge in age/wealth/sex information
         disease_specific_daly_values_this_month = disease_specific_daly_values_this_month.merge(

src/tlo/methods/individual_history_tracker.py

@@ -61,6 +61,7 @@ def initialise_simulation(self, sim):
         notifier.add_listener("hsi_event.pre-run", self.on_event_pre_run)
         notifier.add_listener("hsi_event.post-run", self.on_event_post_run)
         notifier.add_listener("consumables.post-request_consumables", self.on_consumable_request)
+        notifier.add_listener("healthburden.monthly_daly_report", self.on_monthly_daly_report)
 
     def read_parameters(self, resourcefilepath: Optional[Path] = None):
         self.load_parameters_from_dataframe(
@@ -371,6 +372,28 @@ def on_event_post_run(self, data):
         self.consumable_access = {}
         self.cons_call_number_within_event = 0
 
+    def on_monthly_daly_report(self,data):
+        """Upon receiving monthly daly report, convert it to custom EAV format and log"""
+        rows = []
+
+        # This is not a real event, so create custom name and create custom tag associated with it
+        event_name = "monthly_daly_report"
+        event_tag = -1
+
+        for person, dalys in data.items():
+            for cause, value in dalys.items():
+                rows.append({
+                    "entity": person,
+                    "event_name": event_name,
+                    "event_tag": event_tag,
+                    "attribute": cause,
+                    "value": value
+                })
+
+        eav = pd.DataFrame(rows)
+        self.log_eav_dataframe_to_individual_histories(eav)
+        return
+
     def mni_values_differ(self, v1, v2):
 
         if isinstance(v1, list) and isinstance(v2, list):

tests/test_individual_history_tracker.py

@@ -11,11 +11,13 @@
     contraception,
     demography,
     enhanced_lifestyle,
+    healthburden,
     healthseekingbehaviour,
     healthsystem,
     hiv,
     individual_history_tracker,
     labour,
+    malaria,
     newborn_outcomes,
     postnatal_supervisor,
     pregnancy_supervisor,
@@ -56,10 +58,12 @@ def test_individual_history_tracker(tmpdir, seed):
     sim.register(demography.Demography(),
                  enhanced_lifestyle.Lifestyle(),
                  healthsystem.HealthSystem(),
+                 healthburden.HealthBurden(),
                  individual_history_tracker.IndividualHistoryTracker(),
                  symptommanager.SymptomManager(),
                  healthseekingbehaviour.HealthSeekingBehaviour(),
                  chronicsyndrome.ChronicSyndrome(),
+                 malaria.Malaria(),
                  contraception.Contraception(),
                  newborn_outcomes.NewbornOutcomes(),
                  pregnancy_supervisor.PregnancySupervisor(),
@@ -79,6 +83,14 @@ def test_individual_history_tracker(tmpdir, seed):
     output_chains = parse_log_file(sim.log_filepath, level=logging.INFO)
     individual_histories = reconstruct_individual_histories(
                             output_chains['tlo.methods.individual_history_tracker']['individual_histories'])
+
+    # Check that monthly daly reporting is included
+    assert (individual_histories['event_name'] == 'monthly_daly_report').sum() > 0
+
+    # Cannot estimate how many monthly reports should be expected, since monthly report is only logged if individual
+    # experienced dalys that month, so check that at or below this max
+    max_monthly_reports = ((end_date.year - start_date.year) * 12 + (end_date.month - start_date.month))*popsize
+    assert (individual_histories['event_name'] == 'monthly_daly_report').sum() <= max_monthly_reports
 
     # Check that we have a "StartOfSimulation" event for every individual in the initial population,
     #   and that this was logged at the start date
@@ -94,9 +106,10 @@ def test_individual_history_tracker(tmpdir, seed):
     # Assert that all HSI events that occurred were also collected in the event chains.
     # Do not include Inpatient_Care HSIs, as these
     # are not currently treated as being individual-specific
+
     Num_of_HSIs_in_individual_histories = individual_histories["event_name"].str.contains('HSI', na=False).sum()
     Num_of_HSIs_in_hs_log = len(output['tlo.methods.healthsystem']['HSI_Event'].loc[
-    output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care'])
+        output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care'])
     assert Num_of_HSIs_in_individual_histories == Num_of_HSIs_in_hs_log
 
     # Check that aside from HSIs, StartOfSimulation, and Birth, other events were collected too