From aaa0a8493faf0443dfd9bd9289ccae91a0e031a9 Mon Sep 17 00:00:00 2001 From: sd-hystax <110374605+sd-hystax@users.noreply.github.com> Date: Tue, 5 May 2026 12:53:15 +0300 Subject: [PATCH] =?UTF-8?q?OSN-1451.=20Region=20expenses,=20traffic=20expe?= =?UTF-8?q?nses,=20available=20filters=20perfor=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …mance ## Description Region expenses, traffic expenses, available filters performance ## Related issue number OSN-1451 ## Special notes ## Checklist * [ ] The pull request title is a good summary of the changes * [ ] Unit tests for the changes exist * [ ] New and existing unit tests pass locally --- .../controllers/available_filters.py | 60 +++++++++++++++---- .../rest_api_server/controllers/expense.py | 24 ++++---- .../controllers/traffic_expense.py | 2 +- tools/cloud_adapter/clouds/alibaba.py | 4 +- tools/cloud_adapter/clouds/aws.py | 4 +- tools/cloud_adapter/clouds/azure.py | 4 +- tools/cloud_adapter/clouds/base.py | 2 +- tools/cloud_adapter/clouds/databricks.py | 2 +- tools/cloud_adapter/clouds/environment.py | 2 +- tools/cloud_adapter/clouds/gcp.py | 2 +- tools/cloud_adapter/clouds/kubernetes.py | 2 +- tools/cloud_adapter/clouds/nebius.py | 2 +- 12 files changed, 77 insertions(+), 33 deletions(-) diff --git a/rest_api/rest_api_server/controllers/available_filters.py b/rest_api/rest_api_server/controllers/available_filters.py index 2755f7bad..3538b7512 100644 --- a/rest_api/rest_api_server/controllers/available_filters.py +++ b/rest_api/rest_api_server/controllers/available_filters.py @@ -237,23 +237,57 @@ def _aggregate_resource_data(self, match_query, **kwargs): 'day': {'$trunc': { '$divide': ['$first_seen', DAY_IN_SECONDS]}}, }, - 'tags': {'$addToSet': '$tags.k'}, - 'meta': {'$addToSet': '$meta.k'}, + 'tags': {'$push': '$tagKeys'}, + 'meta': {'$push': '$metaKeys'}, 'cloud_resource_ids': {'$addToSet': '$cloud_resource_id'}, }) return self.resources_collection.aggregate([ {'$match': match_query}, - {'$addFields': {'tags': {'$objectToArray': "$tags"}}}, - {'$unwind': { - 'path': "$tags", - 'preserveNullAndEmptyArrays': True - }}, - {'$addFields': {'meta': {'$objectToArray': "$meta"}}}, - {'$unwind': { - 'path': "$meta", - 'preserveNullAndEmptyArrays': True - }}, - {'$group': group_stage} + { + '$addFields': { + 'tagKeys': { + '$map': { + 'input': { + '$objectToArray': {'$ifNull': ['$tags', {}]} + }, + 'as': "t", + 'in': "$$t.k" + } + }, + 'metaKeys': { + '$map': { + 'input': { + '$objectToArray': {'$ifNull': ["$meta", {}]} + }, + 'as': "m", + 'in': "$$m.k" + } + } + } + }, + {'$group': group_stage}, + { + '$addFields': { + 'tags': { + '$reduce': { + 'input': "$tags", + 'initialValue': [], + 'in': {'$setUnion': ["$$value", "$$this"]} + } + } + } + }, + { + '$addFields': { + 'meta': { + '$reduce': { + 'input': "$meta", + 'initialValue': [], + 'in': {'$setUnion': ["$$value", "$$this"]} + } + } + } + } ], allowDiskUse=True) def get(self, organization_id, **params): diff --git a/rest_api/rest_api_server/controllers/expense.py b/rest_api/rest_api_server/controllers/expense.py index 7c5ddc8d9..918f1f31f 100644 --- a/rest_api/rest_api_server/controllers/expense.py +++ b/rest_api/rest_api_server/controllers/expense.py @@ -79,12 +79,16 @@ def _get_expenses_clickhouse( 'deleted_at': 0 }, resource_fields) - external_resource_table = [{ - '_id': x['_id'], - 'cloud_account_id': x['cloud_account_id'], - 'group_field': x.get(resource_field_mappings.get( - group_by, group_by)) - } for x in resource_results if x.get('cloud_account_id')] + cloud_account_ids, external_resource_table = set(), [] + for x in resource_results: + cloud_account_id = x.get('cloud_account_id') + if cloud_account_id: + cloud_account_ids.add(cloud_account_id) + external_resource_table.append({ + '_id': x['_id'], + 'group_field': x.get(resource_field_mappings.get( + group_by, group_by)) + }) expenses_results = self.execute_clickhouse( query=""" SELECT @@ -92,8 +96,8 @@ def _get_expenses_clickhouse( SUM(cost * sign) AS total_cost FROM expenses JOIN resources ON expenses.resource_id = resources._id - AND expenses.cloud_account_id = resources.cloud_account_id - WHERE date >= %(start_date)s + WHERE cloud_account_id IN %(cloud_account_ids)s + AND date >= %(start_date)s AND date <= %(end_date)s GROUP BY date, group_field, cloud_account_id HAVING SUM(sign) > 0 @@ -102,12 +106,12 @@ def _get_expenses_clickhouse( parameters={ 'start_date': start_date, 'end_date': end_date, + 'cloud_account_ids': list(cloud_account_ids), }, external_data=ExternalDataConverter()([{ 'name': 'resources', 'structure': [ ('_id', 'String'), - ('cloud_account_id', 'String'), ('group_field', 'Nullable(String)'), ], 'data': external_resource_table @@ -1982,7 +1986,7 @@ def get_info_map(self, cloud_accs): if cloud_type in scanned: continue adapter = CloudAdapter.get_adapter(cloud_config) - regions = adapter.get_regions_coordinates() + regions = adapter.get_regions_coordinates(load=False) for region_id, info in regions.items(): res.update(self._generate_info_map_element( region_id, info, cloud_type diff --git a/rest_api/rest_api_server/controllers/traffic_expense.py b/rest_api/rest_api_server/controllers/traffic_expense.py index e844059e9..49afd022d 100644 --- a/rest_api/rest_api_server/controllers/traffic_expense.py +++ b/rest_api/rest_api_server/controllers/traffic_expense.py @@ -46,7 +46,7 @@ def _get_configs(cloud_accs): @staticmethod def _get_coordinates(config): cloud_adapter = CloudAdapter.get_adapter(config) - res = cloud_adapter.get_regions_coordinates() + res = cloud_adapter.get_regions_coordinates(load=False) coordinates_result = {} for k, v in res.items(): coordinates = { diff --git a/tools/cloud_adapter/clouds/alibaba.py b/tools/cloud_adapter/clouds/alibaba.py index 907e72d9f..b19a64a9e 100644 --- a/tools/cloud_adapter/clouds/alibaba.py +++ b/tools/cloud_adapter/clouds/alibaba.py @@ -956,9 +956,11 @@ def _get_outdated_regions(): 'longitude': 150.7915495, 'latitude': -33.8481643}, } - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): coordinates_map = self._get_coordinates_map() coordinates_map.update(self._get_outdated_regions()) + if not load: + return coordinates_map try: for region_details in self._list_region_details(): region_id = region_details['RegionId'] diff --git a/tools/cloud_adapter/clouds/aws.py b/tools/cloud_adapter/clouds/aws.py index 78f34b5ab..8e161fad7 100644 --- a/tools/cloud_adapter/clouds/aws.py +++ b/tools/cloud_adapter/clouds/aws.py @@ -1651,12 +1651,14 @@ def _get_coordinates_map(self): 'global': {'longitude': -98.48424, 'latitude': 39.01190} } - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): zero_coordinates = { 'longitude': None, 'latitude': None } coordinates_map = self._get_coordinates_map() + if not load: + return coordinates_map try: for available_region in self.list_regions(): if not coordinates_map.get(available_region): diff --git a/tools/cloud_adapter/clouds/azure.py b/tools/cloud_adapter/clouds/azure.py index bc6791817..4f886eba5 100644 --- a/tools/cloud_adapter/clouds/azure.py +++ b/tools/cloud_adapter/clouds/azure.py @@ -1756,7 +1756,7 @@ def _get_coordinates_map(self): coordinates_map.update(self._get_cn_coordinates_map()) return coordinates_map - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): def to_coord(coordinate): if isinstance(coordinate, str): try: @@ -1766,6 +1766,8 @@ def to_coord(coordinate): return coordinate coordinates_map = self._get_coordinates_map() + if not load: + return coordinates_map try: for region in self.subscription.subscriptions.list_locations( self._subscription_id): diff --git a/tools/cloud_adapter/clouds/base.py b/tools/cloud_adapter/clouds/base.py index 37a7d4891..3d9c1fe0d 100644 --- a/tools/cloud_adapter/clouds/base.py +++ b/tools/cloud_adapter/clouds/base.py @@ -52,7 +52,7 @@ def configure_last_import_modified_at(self): raise NotImplementedError @abc.abstractmethod - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): raise NotImplementedError diff --git a/tools/cloud_adapter/clouds/databricks.py b/tools/cloud_adapter/clouds/databricks.py index 5af1ebaa4..b9c53bdc3 100644 --- a/tools/cloud_adapter/clouds/databricks.py +++ b/tools/cloud_adapter/clouds/databricks.py @@ -217,7 +217,7 @@ def set_currency(self, currency): def configure_last_import_modified_at(self): pass - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): return {} def discovery_calls_map(self): diff --git a/tools/cloud_adapter/clouds/environment.py b/tools/cloud_adapter/clouds/environment.py index 358de736f..5dda399b4 100644 --- a/tools/cloud_adapter/clouds/environment.py +++ b/tools/cloud_adapter/clouds/environment.py @@ -43,7 +43,7 @@ def rds_instance_discovery_calls(self): def ip_address_discovery_calls(self): raise NotImplementedError - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): return {} def set_currency(self, currency): diff --git a/tools/cloud_adapter/clouds/gcp.py b/tools/cloud_adapter/clouds/gcp.py index 9de0dc39b..7d038661f 100644 --- a/tools/cloud_adapter/clouds/gcp.py +++ b/tools/cloud_adapter/clouds/gcp.py @@ -2129,7 +2129,7 @@ def _add_global_coordinates(regions: dict): regions["australia-southeast1"]["alias"] = "Australia" regions["asia-east2"]["alias"] = "China" - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): coordinates = self._get_regions_coordinates() self._add_global_coordinates(coordinates) return coordinates diff --git a/tools/cloud_adapter/clouds/kubernetes.py b/tools/cloud_adapter/clouds/kubernetes.py index 05f95d28d..8143ef08e 100644 --- a/tools/cloud_adapter/clouds/kubernetes.py +++ b/tools/cloud_adapter/clouds/kubernetes.py @@ -230,7 +230,7 @@ def ip_address_discovery_calls(self): # For Kubernetes cloud we don't have ips as a separate resources return [] - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): return {} def set_currency(self, currency): diff --git a/tools/cloud_adapter/clouds/nebius.py b/tools/cloud_adapter/clouds/nebius.py index 6c86ac7df..25c7ec464 100644 --- a/tools/cloud_adapter/clouds/nebius.py +++ b/tools/cloud_adapter/clouds/nebius.py @@ -915,7 +915,7 @@ def get_metric(self, metric_name, instance_ids, start_date, end_date, return self._get_metrics(start_date, end_date, query, folder_id, downsampling) - def get_regions_coordinates(self): + def get_regions_coordinates(self, load=True): return self.config.get('regions_coordinates', {}) or REGIONS_COORDINATES def get_prices(self, currency='USD', filter=None, **kwargs):