From 55adae7bddf9474887e7afe301bc3447182b67ba Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:11:48 +0000 Subject: [PATCH 1/9] Add monthly daly reporting to individual history tracker --- src/tlo/methods/healthburden.py | 16 +++++++++++++ src/tlo/methods/individual_history_tracker.py | 23 +++++++++++++++++++ tests/test_individual_history_tracker.py | 6 ++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/healthburden.py b/src/tlo/methods/healthburden.py index 54db8bf8fb..3cdd867978 100644 --- a/src/tlo/methods/healthburden.py +++ b/src/tlo/methods/healthburden.py @@ -19,6 +19,7 @@ get_gbd_causes_not_represented_in_disease_modules, ) from tlo.methods.demography import age_at_date +from tlo.notify import notifier logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -614,6 +615,21 @@ def apply(self, population): # Multiply 1/12 as these weights are for one month only disease_specific_daly_values_this_month = disease_specific_daly_values_this_month * (1 / 12) + if notifier.has_listeners("healthburden.monthly_dalys_report"): + # Do not dispatch individuals or causes that have zero dalys reported this month + monthly_dalys = disease_specific_daly_values_this_month.copy() + monthly_dalys_nonzero = ( + monthly_dalys.loc[(monthly_dalys != 0).any(axis=1), (monthly_dalys != 0).any(axis=0)] + .to_dict(orient="index")) + + # Store data as dictionary + data = { + person: {col: val for col, val in cols.items() if val != 0} + for person, cols in monthly_dalys_nonzero.items() + } + + notifier.dispatch("healthburden.monthly_dalys_report", data=data) + # 4) Summarise the results for this month wrt sex/age/wealth # - merge in age/wealth/sex information disease_specific_daly_values_this_month = disease_specific_daly_values_this_month.merge( diff --git a/src/tlo/methods/individual_history_tracker.py b/src/tlo/methods/individual_history_tracker.py index 8e70e6ca30..cfe07bdac6 100644 --- a/src/tlo/methods/individual_history_tracker.py +++ b/src/tlo/methods/individual_history_tracker.py @@ -61,6 +61,7 @@ def initialise_simulation(self, sim): notifier.add_listener("hsi_event.pre-run", self.on_event_pre_run) notifier.add_listener("hsi_event.post-run", self.on_event_post_run) notifier.add_listener("consumables.post-request_consumables", self.on_consumable_request) + notifier.add_listener("healthburden.monthly_daly_report", self.on_monthly_daly_report) def read_parameters(self, resourcefilepath: Optional[Path] = None): self.load_parameters_from_dataframe( @@ -371,6 +372,28 @@ def on_event_post_run(self, data): self.consumable_access = {} self.cons_call_number_within_event = 0 + def on_monthly_daly_report(self,data): + """Upon receiving monthly daly report, convert it to custom EAV format and log""" + rows = [] + + # This is not a real event, so create custom name and invalid tag associated with it + event_name = "monthly_daly_report" + event_tag = -1 + + for person, dalys in data.items(): + for cause, value in dalys.items(): + rows.append({ + "entity": person, + "event_name": event_name, + "event_tag": event_tag, + "attribute": cause, + "value": value + }) + + eav = pd.DataFrame(rows) + self.log_eav_dataframe_to_individual_histories(eav) + return + def mni_values_differ(self, v1, v2): if isinstance(v1, list) and isinstance(v2, list): diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index cfe64053f6..b78a0bff35 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -20,6 +20,7 @@ postnatal_supervisor, pregnancy_supervisor, symptommanager, + healthburden, ) resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' @@ -56,6 +57,7 @@ def test_individual_history_tracker(tmpdir, seed): sim.register(demography.Demography(), enhanced_lifestyle.Lifestyle(), healthsystem.HealthSystem(), + healthburden.HealthBurden(), individual_history_tracker.IndividualHistoryTracker(), symptommanager.SymptomManager(), healthseekingbehaviour.HealthSeekingBehaviour(), @@ -79,7 +81,9 @@ def test_individual_history_tracker(tmpdir, seed): output_chains = parse_log_file(sim.log_filepath, level=logging.INFO) individual_histories = reconstruct_individual_histories( output_chains['tlo.methods.individual_history_tracker']['individual_histories']) - + print(individual_histories.columns) + print(individual_histories.loc[individual_histories['event_name']=='monthly_daly_report']) + exit(-1) # Check that we have a "StartOfSimulation" event for every individual in the initial population, #   and that this was logged at the start date assert (individual_histories['event_name'] == 'StartOfSimulation').sum() == popsize From 2bcab124be69594a95c7d228d9fc83bb9753cfee Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:25:02 +0000 Subject: [PATCH 2/9] Fix typo in dispatcher name --- src/tlo/methods/healthburden.py | 7 ++++--- src/tlo/methods/individual_history_tracker.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/tlo/methods/healthburden.py b/src/tlo/methods/healthburden.py index 3cdd867978..b731785b21 100644 --- a/src/tlo/methods/healthburden.py +++ b/src/tlo/methods/healthburden.py @@ -615,7 +615,8 @@ def apply(self, population): # Multiply 1/12 as these weights are for one month only disease_specific_daly_values_this_month = disease_specific_daly_values_this_month * (1 / 12) - if notifier.has_listeners("healthburden.monthly_dalys_report"): + if notifier.has_listeners("healthburden.monthly_daly_report"): + print("Listener is present") # Do not dispatch individuals or causes that have zero dalys reported this month monthly_dalys = disease_specific_daly_values_this_month.copy() monthly_dalys_nonzero = ( @@ -627,8 +628,8 @@ def apply(self, population): person: {col: val for col, val in cols.items() if val != 0} for person, cols in monthly_dalys_nonzero.items() } - - notifier.dispatch("healthburden.monthly_dalys_report", data=data) + print("I will dispatch ", data) + notifier.dispatch("healthburden.monthly_daly_report", data=data) # 4) Summarise the results for this month wrt sex/age/wealth # - merge in age/wealth/sex information diff --git a/src/tlo/methods/individual_history_tracker.py b/src/tlo/methods/individual_history_tracker.py index cfe07bdac6..ce4ac3eac3 100644 --- a/src/tlo/methods/individual_history_tracker.py +++ b/src/tlo/methods/individual_history_tracker.py @@ -374,9 +374,10 @@ def on_event_post_run(self, data): def on_monthly_daly_report(self,data): """Upon receiving monthly daly report, convert it to custom EAV format and log""" + print("I received the dispatch with ", data) rows = [] - # This is not a real event, so create custom name and invalid tag associated with it + # This is not a real event, so create custom name and create custom tag associated with it event_name = "monthly_daly_report" event_tag = -1 From c71299778553470b189d6b6197e3ab1e52a3e112 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:46:49 +0000 Subject: [PATCH 3/9] Finalise test --- tests/test_individual_history_tracker.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index b78a0bff35..2691ff83dd 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -81,9 +81,15 @@ def test_individual_history_tracker(tmpdir, seed): output_chains = parse_log_file(sim.log_filepath, level=logging.INFO) individual_histories = reconstruct_individual_histories( output_chains['tlo.methods.individual_history_tracker']['individual_histories']) - print(individual_histories.columns) - print(individual_histories.loc[individual_histories['event_name']=='monthly_daly_report']) - exit(-1) + + # Check that monthly daly reporting is included + assert (individual_histories['event_name'] == 'monthly_daly_report').sum() > 0 + + # Cannot estimate how many monthly reports should be expected, since monthly report is only logged if individual + # experienced dalys that month, so check that at or below this max + max_monthly_reports = ((end_date.year - start_date.year) * 12 + (end_date.month - start_date.month))*popsize + assert (individual_histories['event_name'] == 'monthly_daly_report').sum() <= max_monthly_reports + # Check that we have a "StartOfSimulation" event for every individual in the initial population, #   and that this was logged at the start date assert (individual_histories['event_name'] == 'StartOfSimulation').sum() == popsize From 6ae6bfd4ff06bcb944dbdd9dccdfc0c945f12e52 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:59:31 +0000 Subject: [PATCH 4/9] Style fix --- tests/test_individual_history_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index 2691ff83dd..31e1090bdd 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -11,6 +11,7 @@ contraception, demography, enhanced_lifestyle, + healthburden, healthseekingbehaviour, healthsystem, hiv, @@ -20,7 +21,6 @@ postnatal_supervisor, pregnancy_supervisor, symptommanager, - healthburden, ) resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' From a3e43bff04e40a0e83d4f45865474f395e64a0be Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 18 Mar 2026 15:41:38 +0000 Subject: [PATCH 5/9] Remove debug messaging --- src/tlo/methods/healthburden.py | 2 -- src/tlo/methods/individual_history_tracker.py | 1 - 2 files changed, 3 deletions(-) diff --git a/src/tlo/methods/healthburden.py b/src/tlo/methods/healthburden.py index b731785b21..083644226c 100644 --- a/src/tlo/methods/healthburden.py +++ b/src/tlo/methods/healthburden.py @@ -616,7 +616,6 @@ def apply(self, population): disease_specific_daly_values_this_month = disease_specific_daly_values_this_month * (1 / 12) if notifier.has_listeners("healthburden.monthly_daly_report"): - print("Listener is present") # Do not dispatch individuals or causes that have zero dalys reported this month monthly_dalys = disease_specific_daly_values_this_month.copy() monthly_dalys_nonzero = ( @@ -628,7 +627,6 @@ def apply(self, population): person: {col: val for col, val in cols.items() if val != 0} for person, cols in monthly_dalys_nonzero.items() } - print("I will dispatch ", data) notifier.dispatch("healthburden.monthly_daly_report", data=data) # 4) Summarise the results for this month wrt sex/age/wealth diff --git a/src/tlo/methods/individual_history_tracker.py b/src/tlo/methods/individual_history_tracker.py index ce4ac3eac3..f4bdf13d39 100644 --- a/src/tlo/methods/individual_history_tracker.py +++ b/src/tlo/methods/individual_history_tracker.py @@ -374,7 +374,6 @@ def on_event_post_run(self, data): def on_monthly_daly_report(self,data): """Upon receiving monthly daly report, convert it to custom EAV format and log""" - print("I received the dispatch with ", data) rows = [] # This is not a real event, so create custom name and create custom tag associated with it From cf79a8e8da5b72791e956ce27763e2266a5b9dd6 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 19 Mar 2026 09:19:04 +0000 Subject: [PATCH 6/9] Simplify dictionary conversion --- src/tlo/methods/healthburden.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tlo/methods/healthburden.py b/src/tlo/methods/healthburden.py index 083644226c..3b12fc43f9 100644 --- a/src/tlo/methods/healthburden.py +++ b/src/tlo/methods/healthburden.py @@ -617,16 +617,16 @@ def apply(self, population): if notifier.has_listeners("healthburden.monthly_daly_report"): # Do not dispatch individuals or causes that have zero dalys reported this month - monthly_dalys = disease_specific_daly_values_this_month.copy() + monthly_dalys = disease_specific_daly_values_this_month monthly_dalys_nonzero = ( - monthly_dalys.loc[(monthly_dalys != 0).any(axis=1), (monthly_dalys != 0).any(axis=0)] - .to_dict(orient="index")) + monthly_dalys.loc[(monthly_dalys != 0).any(axis=1), (monthly_dalys != 0).any(axis=0)]) - # Store data as dictionary + # Only retain non-zero info data = { - person: {col: val for col, val in cols.items() if val != 0} - for person, cols in monthly_dalys_nonzero.items() + idx: {col: val for col, val in row.items() if val > 0} + for idx, row in monthly_dalys_nonzero.iterrows() } + notifier.dispatch("healthburden.monthly_daly_report", data=data) # 4) Summarise the results for this month wrt sex/age/wealth From dffe72902380246a7331d84953ec281c6e1a482f Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:11:03 +0000 Subject: [PATCH 7/9] Investigate cause of discrepancy between HSI log and iht --- tests/test_individual_history_tracker.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index 31e1090bdd..c222be0476 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from collections import Counter from tlo import Date, Simulation, logging from tlo.analysis.utils import parse_log_file, reconstruct_individual_histories @@ -17,9 +18,12 @@ hiv, individual_history_tracker, labour, + malaria, newborn_outcomes, postnatal_supervisor, pregnancy_supervisor, + rti, + schisto, symptommanager, ) @@ -62,6 +66,9 @@ def test_individual_history_tracker(tmpdir, seed): symptommanager.SymptomManager(), healthseekingbehaviour.HealthSeekingBehaviour(), chronicsyndrome.ChronicSyndrome(), + #malaria.Malaria(), + rti.RTI(), + schisto.Schisto(), contraception.Contraception(), newborn_outcomes.NewbornOutcomes(), pregnancy_supervisor.PregnancySupervisor(), @@ -107,6 +114,28 @@ def test_individual_history_tracker(tmpdir, seed): Num_of_HSIs_in_individual_histories = individual_histories["event_name"].str.contains('HSI', na=False).sum() Num_of_HSIs_in_hs_log = len(output['tlo.methods.healthsystem']['HSI_Event'].loc[ output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care']) + + + print("HSIs in log") + HSIs_in_log = output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care', 'Event_Name'].to_list() + HSIs_in_IHT = individual_histories.loc[individual_histories["event_name"].str.contains('HSI', na=False),"event_name"].tolist() + + only_in_log = list((Counter(HSIs_in_log) - Counter(HSIs_in_IHT)).elements()) + only_in_IHT = list((Counter(HSIs_in_IHT) - Counter(HSIs_in_log)).elements()) + print("Only in log") + print(only_in_log) + print("Only in IHT") + print(only_in_IHT) + + output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment'].to_csv('HSI_event_log.csv') + individual_histories.loc[individual_histories["event_name"].str.contains('HSI_Malaria_Treatment', na=False)].to_csv('HSI_event_IHT.csv') + individual_histories.to_csv('full_IHT.csv') + + malaria_events_in_log =output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment'] + malaria_events_in_IHT = individual_histories.loc[individual_histories["event_name"].str.contains('HSI_Malaria_Treatment', na=False)] + + + print(output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment']) assert Num_of_HSIs_in_individual_histories == Num_of_HSIs_in_hs_log # Check that aside from HSIs, StartOfSimulation, and Birth, other events were collected too From 9e094db5d9280a2e2da84f893ff4823dbd57aae1 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:55:34 +0000 Subject: [PATCH 8/9] Fix lack of event tag usage --- src/tlo/analysis/utils.py | 8 ++---- tests/test_individual_history_tracker.py | 31 ++++-------------------- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index 6b4d2cbf9b..6c3fd9db0f 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -413,14 +413,14 @@ def reconstruct_individual_histories(df): # Collapse into 'entity', 'date', 'event_name', 'Info' format where 'Info' is dict listing attributes # (e.g. {a1:v1, a2:v2, a3:v3, ...} ) df_collapsed = ( - df.groupby(['entity', 'date', 'event_name'], sort=False) + df.groupby(['entity', 'date', 'event_name', 'event_tag'], sort=False) .apply(lambda g: dict(zip(g['attribute'], g['value']))) .reset_index(name='Info') ) df_final = ( df_collapsed - .sort_values(by=['entity', 'date']) + .sort_values(by=['entity', 'date', 'event_tag']) .reset_index(drop=True) ) @@ -430,8 +430,6 @@ def reconstruct_individual_histories(df): if len(problems)>0: print("Values didn't change but were still detected") print(problems) - - return df_final @@ -485,8 +483,6 @@ def extract_individual_histories(results_folder: Path, # Combine all dfs into a single DataFrame res[draw] = pd.concat(dfs_from_runs, ignore_index=True) - res[0].to_csv('individual_histories.csv') - return res diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index c222be0476..04e272bd9d 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -66,9 +66,9 @@ def test_individual_history_tracker(tmpdir, seed): symptommanager.SymptomManager(), healthseekingbehaviour.HealthSeekingBehaviour(), chronicsyndrome.ChronicSyndrome(), - #malaria.Malaria(), - rti.RTI(), - schisto.Schisto(), + malaria.Malaria(), + #rti.RTI(), + #schisto.Schisto(), contraception.Contraception(), newborn_outcomes.NewbornOutcomes(), pregnancy_supervisor.PregnancySupervisor(), @@ -111,31 +111,10 @@ def test_individual_history_tracker(tmpdir, seed): # Assert that all HSI events that occurred were also collected in the event chains. # Do not include Inpatient_Care HSIs, as these # are not currently treated as being individual-specific + Num_of_HSIs_in_individual_histories = individual_histories["event_name"].str.contains('HSI', na=False).sum() Num_of_HSIs_in_hs_log = len(output['tlo.methods.healthsystem']['HSI_Event'].loc[ - output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care']) - - - print("HSIs in log") - HSIs_in_log = output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care', 'Event_Name'].to_list() - HSIs_in_IHT = individual_histories.loc[individual_histories["event_name"].str.contains('HSI', na=False),"event_name"].tolist() - - only_in_log = list((Counter(HSIs_in_log) - Counter(HSIs_in_IHT)).elements()) - only_in_IHT = list((Counter(HSIs_in_IHT) - Counter(HSIs_in_log)).elements()) - print("Only in log") - print(only_in_log) - print("Only in IHT") - print(only_in_IHT) - - output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment'].to_csv('HSI_event_log.csv') - individual_histories.loc[individual_histories["event_name"].str.contains('HSI_Malaria_Treatment', na=False)].to_csv('HSI_event_IHT.csv') - individual_histories.to_csv('full_IHT.csv') - - malaria_events_in_log =output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment'] - malaria_events_in_IHT = individual_histories.loc[individual_histories["event_name"].str.contains('HSI_Malaria_Treatment', na=False)] - - - print(output['tlo.methods.healthsystem']['HSI_Event'].loc[output['tlo.methods.healthsystem']['HSI_Event']['Event_Name']=='HSI_Malaria_Treatment']) + output['tlo.methods.healthsystem']['HSI_Event']['Event_Name'] != 'Inpatient_Care']) assert Num_of_HSIs_in_individual_histories == Num_of_HSIs_in_hs_log # Check that aside from HSIs, StartOfSimulation, and Birth, other events were collected too From a16a4c45f5f4e7b2952c3cc6e7ee2232d639efa1 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 20 Mar 2026 10:05:46 +0000 Subject: [PATCH 9/9] Style fix --- tests/test_individual_history_tracker.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_individual_history_tracker.py b/tests/test_individual_history_tracker.py index 04e272bd9d..f01968b895 100644 --- a/tests/test_individual_history_tracker.py +++ b/tests/test_individual_history_tracker.py @@ -2,7 +2,6 @@ from pathlib import Path import pytest -from collections import Counter from tlo import Date, Simulation, logging from tlo.analysis.utils import parse_log_file, reconstruct_individual_histories @@ -22,8 +21,6 @@ newborn_outcomes, postnatal_supervisor, pregnancy_supervisor, - rti, - schisto, symptommanager, ) @@ -67,8 +64,6 @@ def test_individual_history_tracker(tmpdir, seed): healthseekingbehaviour.HealthSeekingBehaviour(), chronicsyndrome.ChronicSyndrome(), malaria.Malaria(), - #rti.RTI(), - #schisto.Schisto(), contraception.Contraception(), newborn_outcomes.NewbornOutcomes(), pregnancy_supervisor.PregnancySupervisor(),