From 16fcb2ae4e27a468149ceef60ed3d62155c4cd3a Mon Sep 17 00:00:00 2001 From: nicolasrbr Date: Tue, 17 Mar 2026 15:15:03 -0300 Subject: [PATCH 1/4] add: logs for abandoned S3 buckets --- .../bumiworker/modules/abandoned_base.py | 14 +++++++++-- .../recommendations/s3_abandoned_buckets.py | 25 +++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/bumiworker/bumiworker/modules/abandoned_base.py b/bumiworker/bumiworker/modules/abandoned_base.py index 8650540b4..16a24f74e 100644 --- a/bumiworker/bumiworker/modules/abandoned_base.py +++ b/bumiworker/bumiworker/modules/abandoned_base.py @@ -78,8 +78,18 @@ def _get_data_size_request_metrics(self, cloud_account_id, def _are_below_thresholds(res_data_request_map, metric_threshold_map): resource_ids = [] for res_id, data_request_map in res_data_request_map.items(): - if all(data_request_map.get(key, 0) <= threshold_value - for key, threshold_value in metric_threshold_map.items()): + checks = { + key: data_request_map.get(key, 0) <= threshold_value + for key, threshold_value in metric_threshold_map.items() + } + LOG.debug( + 'AB - Resource %s threshold checks: values=%s thresholds=%s checks=%s', + res_id, + data_request_map, + metric_threshold_map, + checks + ) + if all(checks.values()): resource_ids.append(res_id) LOG.debug(f'AB - Resources below thresholds: {resource_ids}') return resource_ids diff --git a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py index cb0544b02..1120b2fb2 100644 --- a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py +++ b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py @@ -30,11 +30,14 @@ def __init__(self, organization_id, config_client, created_at): def get_metric_threshold_map(self): # Buckets are considered abandoned if both GetObject and PutObject # operations are zero (no read or write activity) - LOG.debug(f'AB - GET_OBJECT_KEY: {GET_OBJECT_KEY}, PUT_OBJECT_KEY: {PUT_OBJECT_KEY}') - return { + metric_threshold_map = { GET_OBJECT_KEY: False, PUT_OBJECT_KEY: False } + LOG.info( + 'AB - Metric threshold map loaded: %s (abandoned when both false)', + metric_threshold_map) + return metric_threshold_map def _get_data_size_request_metrics(self, cloud_account_id, cloud_resource_ids, start_date, @@ -68,7 +71,6 @@ def _get_data_size_request_metrics(self, cloud_account_id, api_requests = self.mongo_client.restapi.raw_expenses.aggregate( api_request_pipeline) api_requests_list = list(api_requests) - LOG.debug(f'AB - API Requests aggregation result: {api_requests_list}') resource_meter_value = {} # Initialize all resources with no recorded activity for res_id in cloud_resource_ids: @@ -89,15 +91,28 @@ def _get_data_size_request_metrics(self, cloud_account_id, resource_meter_value[cloud_resource_id][ PUT_OBJECT_KEY] = has_usage - LOG.debug(f'AB - Resource meter values: {resource_meter_value}') + for resource_id, meter_values in resource_meter_value.items(): + LOG.info( + 'AB - Resource %s operations status: get_object=%s put_object=%s', + resource_id, + meter_values[GET_OBJECT_KEY], + meter_values[PUT_OBJECT_KEY], + ) + + LOG.info(f'AB - Resource meter values: {resource_meter_value}') return resource_meter_value @staticmethod def metrics_result(data_req_map): - return { + result = { 'get_object_count': data_req_map.get(GET_OBJECT_KEY, False), 'put_object_count': data_req_map.get(PUT_OBJECT_KEY, False), } + LOG.info( + 'AB - metrics_result mapping: %s -> %s', + data_req_map, + result) + return result def main(organization_id, config_client, created_at, **kwargs): From 6b77a0c20e83477947d80521e8d6810b29db2e09 Mon Sep 17 00:00:00 2001 From: nicolasrbr Date: Thu, 19 Mar 2026 17:01:13 -0300 Subject: [PATCH 2/4] add: logs for abandoned s3 bucket --- bumiworker/bumiworker/modules/abandoned_base.py | 16 +++------------- .../recommendations/s3_abandoned_buckets.py | 14 +++----------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/bumiworker/bumiworker/modules/abandoned_base.py b/bumiworker/bumiworker/modules/abandoned_base.py index 16a24f74e..c9cfd3a46 100644 --- a/bumiworker/bumiworker/modules/abandoned_base.py +++ b/bumiworker/bumiworker/modules/abandoned_base.py @@ -78,20 +78,10 @@ def _get_data_size_request_metrics(self, cloud_account_id, def _are_below_thresholds(res_data_request_map, metric_threshold_map): resource_ids = [] for res_id, data_request_map in res_data_request_map.items(): - checks = { - key: data_request_map.get(key, 0) <= threshold_value - for key, threshold_value in metric_threshold_map.items() - } - LOG.debug( - 'AB - Resource %s threshold checks: values=%s thresholds=%s checks=%s', - res_id, - data_request_map, - metric_threshold_map, - checks - ) - if all(checks.values()): + if all(data_request_map.get(key, 0) <= threshold_value + for key, threshold_value in metric_threshold_map.items()): resource_ids.append(res_id) - LOG.debug(f'AB - Resources below thresholds: {resource_ids}') + LOG.info(f'AB - Resources below thresholds: {resource_ids}') return resource_ids @staticmethod diff --git a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py index 1120b2fb2..864845034 100644 --- a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py +++ b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py @@ -30,14 +30,11 @@ def __init__(self, organization_id, config_client, created_at): def get_metric_threshold_map(self): # Buckets are considered abandoned if both GetObject and PutObject # operations are zero (no read or write activity) - metric_threshold_map = { + LOG.info(f'AB - GET_OBJECT_KEY: {GET_OBJECT_KEY}, PUT_OBJECT_KEY: {PUT_OBJECT_KEY}') + return { GET_OBJECT_KEY: False, PUT_OBJECT_KEY: False } - LOG.info( - 'AB - Metric threshold map loaded: %s (abandoned when both false)', - metric_threshold_map) - return metric_threshold_map def _get_data_size_request_metrics(self, cloud_account_id, cloud_resource_ids, start_date, @@ -104,15 +101,10 @@ def _get_data_size_request_metrics(self, cloud_account_id, @staticmethod def metrics_result(data_req_map): - result = { + return { 'get_object_count': data_req_map.get(GET_OBJECT_KEY, False), 'put_object_count': data_req_map.get(PUT_OBJECT_KEY, False), } - LOG.info( - 'AB - metrics_result mapping: %s -> %s', - data_req_map, - result) - return result def main(organization_id, config_client, created_at, **kwargs): From 9277a986bb90976a4ce96c050884137ff5dc96e8 Mon Sep 17 00:00:00 2001 From: nicolasrbr Date: Wed, 25 Mar 2026 16:43:13 -0300 Subject: [PATCH 3/4] fix Abandoned s3 bucket api request --- .../modules/recommendations/s3_abandoned_buckets.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py index 864845034..8944ac94f 100644 --- a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py +++ b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py @@ -30,7 +30,6 @@ def __init__(self, organization_id, config_client, created_at): def get_metric_threshold_map(self): # Buckets are considered abandoned if both GetObject and PutObject # operations are zero (no read or write activity) - LOG.info(f'AB - GET_OBJECT_KEY: {GET_OBJECT_KEY}, PUT_OBJECT_KEY: {PUT_OBJECT_KEY}') return { GET_OBJECT_KEY: False, PUT_OBJECT_KEY: False @@ -59,8 +58,8 @@ def _get_data_size_request_metrics(self, cloud_account_id, '_id': '$resource_id', 'operation': '$lineItem/Operation' }, - 'total_usage': { - '$sum': '$lineItem/UsageAmount' + 'usage_amount': { + '$push': '$lineItem/UsageAmount' } } } @@ -75,11 +74,13 @@ def _get_data_size_request_metrics(self, cloud_account_id, GET_OBJECT_KEY: False, PUT_OBJECT_KEY: False } - # Aggregate operation usage (already summed by MongoDB) + # Aggregate operation usage by summing the pushed amounts for api_request in api_requests_list: cloud_resource_id = api_request['_id']['_id'] operation = api_request['_id']['operation'] - total_sum = int(api_request['total_usage']) + usage_amounts = api_request['usage_amount'] + total_sum = sum(float(amount) for amount in usage_amounts) if usage_amounts else 0 + LOG.info(f'AB - Resource {cloud_resource_id} operation {operation} total usage amount: {total_sum}') has_usage = bool(total_sum) if operation == 'GetObject': resource_meter_value[cloud_resource_id][ From ab88118c9ccdefca99ceea626f71e22ab2ce24b8 Mon Sep 17 00:00:00 2001 From: nicolasrbr Date: Thu, 26 Mar 2026 09:29:47 -0300 Subject: [PATCH 4/4] fix abandoned s3 buckets --- bumiworker/bumiworker/modules/abandoned_base.py | 7 +------ .../modules/recommendations/s3_abandoned_buckets.py | 10 ---------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/bumiworker/bumiworker/modules/abandoned_base.py b/bumiworker/bumiworker/modules/abandoned_base.py index c9cfd3a46..48fa01616 100644 --- a/bumiworker/bumiworker/modules/abandoned_base.py +++ b/bumiworker/bumiworker/modules/abandoned_base.py @@ -1,7 +1,7 @@ from collections import defaultdict from datetime import datetime, timedelta, timezone from bumiworker.bumiworker.modules.base import ( - LOG, ArchiveBase, ArchiveReason, ModuleBase, DAYS_IN_MONTH + ArchiveBase, ArchiveReason, ModuleBase, DAYS_IN_MONTH ) from tools.optscale_data.clickhouse import ExternalDataConverter from tools.optscale_time import utcnow, startday @@ -81,7 +81,6 @@ def _are_below_thresholds(res_data_request_map, metric_threshold_map): if all(data_request_map.get(key, 0) <= threshold_value for key, threshold_value in metric_threshold_map.items()): resource_ids.append(res_id) - LOG.info(f'AB - Resources below thresholds: {resource_ids}') return resource_ids @staticmethod @@ -107,8 +106,6 @@ def _get(self): now = utcnow() start_date = now - timedelta(days=self.days_threshold) - LOG.debug(f'AB - Start date for abandoned buckets: {start_date}') - cloud_accounts = self.get_cloud_accounts(self.SUPPORTED_CLOUD_TYPES, self.skip_cloud_accounts) buckets_by_account = self.get_active_resources( @@ -163,8 +160,6 @@ def _get(self): self.metrics_result(data_req_map)) result.append(base_result_dict) - LOG.debug(f'AB - Abandoned buckets result: {result}') - return result diff --git a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py index 8944ac94f..9ae3a112e 100644 --- a/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py +++ b/bumiworker/bumiworker/modules/recommendations/s3_abandoned_buckets.py @@ -80,7 +80,6 @@ def _get_data_size_request_metrics(self, cloud_account_id, operation = api_request['_id']['operation'] usage_amounts = api_request['usage_amount'] total_sum = sum(float(amount) for amount in usage_amounts) if usage_amounts else 0 - LOG.info(f'AB - Resource {cloud_resource_id} operation {operation} total usage amount: {total_sum}') has_usage = bool(total_sum) if operation == 'GetObject': resource_meter_value[cloud_resource_id][ @@ -89,15 +88,6 @@ def _get_data_size_request_metrics(self, cloud_account_id, resource_meter_value[cloud_resource_id][ PUT_OBJECT_KEY] = has_usage - for resource_id, meter_values in resource_meter_value.items(): - LOG.info( - 'AB - Resource %s operations status: get_object=%s put_object=%s', - resource_id, - meter_values[GET_OBJECT_KEY], - meter_values[PUT_OBJECT_KEY], - ) - - LOG.info(f'AB - Resource meter values: {resource_meter_value}') return resource_meter_value @staticmethod