Skip to content

Pull request update/260527 #885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stanfra merged 1 commit into from
May 27, 2026
Merged

Pull request update/260527 #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 47 additions & 13 deletions rest_api/rest_api_server/controllers/available_filters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,23 +237,57 @@ def _aggregate_resource_data(self, match_query, **kwargs):
'day': {'$trunc': {
'$divide': ['$first_seen', DAY_IN_SECONDS]}},
},
'tags': {'$addToSet': '$tags.k'},
'meta': {'$addToSet': '$meta.k'},
'tags': {'$push': '$tagKeys'},
'meta': {'$push': '$metaKeys'},
'cloud_resource_ids': {'$addToSet': '$cloud_resource_id'},
})
return self.resources_collection.aggregate([
{'$match': match_query},
{'$addFields': {'tags': {'$objectToArray': "$tags"}}},
{'$unwind': {
'path': "$tags",
'preserveNullAndEmptyArrays': True
}},
{'$addFields': {'meta': {'$objectToArray': "$meta"}}},
{'$unwind': {
'path': "$meta",
'preserveNullAndEmptyArrays': True
}},
{'$group': group_stage}
{
'$addFields': {
'tagKeys': {
'$map': {
'input': {
'$objectToArray': {'$ifNull': ['$tags', {}]}
},
'as': "t",
'in': "$$t.k"
}
},
'metaKeys': {
'$map': {
'input': {
'$objectToArray': {'$ifNull': ["$meta", {}]}
},
'as': "m",
'in': "$$m.k"
}
}
}
},
{'$group': group_stage},
{
'$addFields': {
'tags': {
'$reduce': {
'input': "$tags",
'initialValue': [],
'in': {'$setUnion': ["$$value", "$$this"]}
}
}
}
},
{
'$addFields': {
'meta': {
'$reduce': {
'input': "$meta",
'initialValue': [],
'in': {'$setUnion': ["$$value", "$$this"]}
}
}
}
}
], allowDiskUse=True)

def get(self, organization_id, **params):
Expand Down
24 changes: 14 additions & 10 deletions rest_api/rest_api_server/controllers/expense.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,21 +79,25 @@ def _get_expenses_clickhouse(
'deleted_at': 0
}, resource_fields)

external_resource_table = [{
'_id': x['_id'],
'cloud_account_id': x['cloud_account_id'],
'group_field': x.get(resource_field_mappings.get(
group_by, group_by))
} for x in resource_results if x.get('cloud_account_id')]
cloud_account_ids, external_resource_table = set(), []
for x in resource_results:
cloud_account_id = x.get('cloud_account_id')
if cloud_account_id:
cloud_account_ids.add(cloud_account_id)
external_resource_table.append({
'_id': x['_id'],
'group_field': x.get(resource_field_mappings.get(
group_by, group_by))
})
expenses_results = self.execute_clickhouse(
query="""
SELECT
date, group_field, cloud_account_id,
SUM(cost * sign) AS total_cost
FROM expenses
JOIN resources ON expenses.resource_id = resources._id
AND expenses.cloud_account_id = resources.cloud_account_id
WHERE date >= %(start_date)s
WHERE cloud_account_id IN %(cloud_account_ids)s
AND date >= %(start_date)s
AND date <= %(end_date)s
GROUP BY date, group_field, cloud_account_id
HAVING SUM(sign) > 0
Expand All @@ -102,12 +106,12 @@ def _get_expenses_clickhouse(
parameters={
'start_date': start_date,
'end_date': end_date,
'cloud_account_ids': list(cloud_account_ids),
},
external_data=ExternalDataConverter()([{
'name': 'resources',
'structure': [
('_id', 'String'),
('cloud_account_id', 'String'),
('group_field', 'Nullable(String)'),
],
'data': external_resource_table
Expand Down Expand Up @@ -1982,7 +1986,7 @@ def get_info_map(self, cloud_accs):
if cloud_type in scanned:
continue
adapter = CloudAdapter.get_adapter(cloud_config)
regions = adapter.get_regions_coordinates()
regions = adapter.get_regions_coordinates(load=False)
for region_id, info in regions.items():
res.update(self._generate_info_map_element(
region_id, info, cloud_type
Expand Down
2 changes: 1 addition & 1 deletion rest_api/rest_api_server/controllers/traffic_expense.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def _get_configs(cloud_accs):
@staticmethod
def _get_coordinates(config):
cloud_adapter = CloudAdapter.get_adapter(config)
res = cloud_adapter.get_regions_coordinates()
res = cloud_adapter.get_regions_coordinates(load=False)
coordinates_result = {}
for k, v in res.items():
coordinates = {
Expand Down
4 changes: 3 additions & 1 deletion tools/cloud_adapter/clouds/alibaba.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -956,9 +956,11 @@ def _get_outdated_regions():
'longitude': 150.7915495, 'latitude': -33.8481643},
}

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
coordinates_map = self._get_coordinates_map()
coordinates_map.update(self._get_outdated_regions())
if not load:
return coordinates_map
try:
for region_details in self._list_region_details():
region_id = region_details['RegionId']
Expand Down
4 changes: 3 additions & 1 deletion tools/cloud_adapter/clouds/aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1651,12 +1651,14 @@ def _get_coordinates_map(self):
'global': {'longitude': -98.48424, 'latitude': 39.01190}
}

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
zero_coordinates = {
'longitude': None,
'latitude': None
}
coordinates_map = self._get_coordinates_map()
if not load:
return coordinates_map
try:
for available_region in self.list_regions():
if not coordinates_map.get(available_region):
Expand Down
4 changes: 3 additions & 1 deletion tools/cloud_adapter/clouds/azure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1756,7 +1756,7 @@ def _get_coordinates_map(self):
coordinates_map.update(self._get_cn_coordinates_map())
return coordinates_map

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
def to_coord(coordinate):
if isinstance(coordinate, str):
try:
Expand All @@ -1766,6 +1766,8 @@ def to_coord(coordinate):
return coordinate

coordinates_map = self._get_coordinates_map()
if not load:
return coordinates_map
try:
for region in self.subscription.subscriptions.list_locations(
self._subscription_id):
Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def configure_last_import_modified_at(self):
raise NotImplementedError

@abc.abstractmethod
def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
raise NotImplementedError


Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/databricks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def set_currency(self, currency):
def configure_last_import_modified_at(self):
pass

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
return {}

def discovery_calls_map(self):
Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def rds_instance_discovery_calls(self):
def ip_address_discovery_calls(self):
raise NotImplementedError

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
return {}

def set_currency(self, currency):
Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/gcp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2129,7 +2129,7 @@ def _add_global_coordinates(regions: dict):
regions["australia-southeast1"]["alias"] = "Australia"
regions["asia-east2"]["alias"] = "China"

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
coordinates = self._get_regions_coordinates()
self._add_global_coordinates(coordinates)
return coordinates
Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/kubernetes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ def ip_address_discovery_calls(self):
# For Kubernetes cloud we don't have ips as a separate resources
return []

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
return {}

def set_currency(self, currency):
Expand Down
2 changes: 1 addition & 1 deletion tools/cloud_adapter/clouds/nebius.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,7 @@ def get_metric(self, metric_name, instance_ids, start_date, end_date,
return self._get_metrics(start_date, end_date, query, folder_id,
downsampling)

def get_regions_coordinates(self):
def get_regions_coordinates(self, load=True):
return self.config.get('regions_coordinates', {}) or REGIONS_COORDINATES

def get_prices(self, currency='USD', filter=None, **kwargs):
Expand Down