diff --git a/interfaces/otlp/CHANGELOG.md b/interfaces/otlp/CHANGELOG.md index c236b2e8..8fa1859f 100644 --- a/interfaces/otlp/CHANGELOG.md +++ b/interfaces/otlp/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `gRPC` is favoured over `HTTP` when multiple endpoints are available - Support for publishing LogQL and PromQL rules via `OtlpRequirer.publish()` - LZMA+base64 compression of rules in the requirer databag to avoid Juju databag size limits -- `OtlpProvider.rules()` for fetching rules from all requirer relations with injected Juju topology and validation +- `OtlpProvider.rules()` for fetching rules from all requirer relations with injected Juju topology and validation - Generic aggregator rules automatically included in every requirer's published rule set - Python 3.10+ compatibility @@ -26,3 +26,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Updated - Replace the requirer's rule path interface with an interface accepting an object containing rules +- Generic PromQL alert rules for requirer charms that are of the aggregator or application type diff --git a/interfaces/otlp/src/charmlibs/interfaces/otlp/__init__.py b/interfaces/otlp/src/charmlibs/interfaces/otlp/__init__.py index 7624dfee..3584680b 100644 --- a/interfaces/otlp/src/charmlibs/interfaces/otlp/__init__.py +++ b/interfaces/otlp/src/charmlibs/interfaces/otlp/__init__.py @@ -115,6 +115,13 @@ def _publish_rules(self, _: ops.EventBase): ) OtlpRequirer(self, rules=rules).publish() +Generic rules are sourced from `cosl.rules.generic_alert_groups `_. +If the charm is an aggregator e.g., opentelemetry-collector, the type of generic rules to be +injected into the charm's RuleStore should reflect that. This is configurable by setting the +``aggregator_peer_relation_name`` with the name of the charm's peer relation:: + + OtlpRequirer(..., aggregator_peer_relation_name="my-peers").publish() + Relation Data Format ==================== diff --git a/interfaces/otlp/src/charmlibs/interfaces/otlp/_otlp.py b/interfaces/otlp/src/charmlibs/interfaces/otlp/_otlp.py index 2a484f5d..1d89e532 100644 --- a/interfaces/otlp/src/charmlibs/interfaces/otlp/_otlp.py +++ b/interfaces/otlp/src/charmlibs/interfaces/otlp/_otlp.py @@ -22,6 +22,7 @@ import copy import json import logging +import re from collections import OrderedDict from collections.abc import Sequence from dataclasses import dataclass, field @@ -29,8 +30,13 @@ from typing import Any, Final, Literal from cosl.juju_topology import JujuTopology -from cosl.rules import InjectResult, Rules, generic_alert_groups -from cosl.types import OfficialRuleFileFormat +from cosl.rules import ( + HOST_METRICS_MISSING_RULE_NAME, + InjectResult, + Rules, + generic_alert_groups, +) +from cosl.types import OfficialRuleFileFormat, SingleRuleFormat from cosl.utils import LZMABase64 from ops import CharmBase from pydantic import ( @@ -64,29 +70,63 @@ def __post_init__(self): def add_logql( self, - rule_dict: dict[str, Any], + rule_dict: OfficialRuleFileFormat | SingleRuleFormat, *, group_name: str | None = None, group_name_prefix: str | None = None, ) -> 'RuleStore': + """Add rules from dict to the existing LogQL ruleset. + + Args: + rule_dict: a single-rule or official-rule YAML dict + group_name: a custom group name, used only if the new rule is of single-rule format + group_name_prefix: a custom group name prefix, used only if the new rule is of + single-rule format + """ self.logql.add(rule_dict, group_name=group_name, group_name_prefix=group_name_prefix) return self def add_logql_path(self, dir_path: str | Path, *, recursive: bool = False) -> 'RuleStore': + """Add LogQL rules from a dir path. + + All rules from files are aggregated into a data structure representing a single rule file. + All group names are augmented with juju topology. + + Args: + dir_path: either a rules file or a dir of rules files. + recursive: whether to read files recursively or not (no impact if `path` is a file). + """ self.logql.add_path(dir_path, recursive=recursive) return self def add_promql( self, - rule_dict: dict[str, Any], + rule_dict: OfficialRuleFileFormat | SingleRuleFormat, *, group_name: str | None = None, group_name_prefix: str | None = None, ) -> 'RuleStore': + """Add rules from dict to the existing PromQL ruleset. + + Args: + rule_dict: a single-rule or official-rule YAML dict + group_name: a custom group name, used only if the new rule is of single-rule format + group_name_prefix: a custom group name prefix, used only if the new rule is of + single-rule format + """ self.promql.add(rule_dict, group_name=group_name, group_name_prefix=group_name_prefix) return self def add_promql_path(self, dir_path: str | Path, *, recursive: bool = False) -> 'RuleStore': + """Add PromQL rules from a dir path. + + All rules from files are aggregated into a data structure representing a single rule file. + All group names are augmented with juju topology. + + Args: + dir_path: either a rules file or a dir of rules files. + recursive: whether to read files recursively or not (no impact if `path` is a file). + """ self.promql.add_path(dir_path, recursive=recursive) return self @@ -174,6 +214,10 @@ class OtlpRequirer: endpoints. telemetries: The telemetries to filter for in the provider's OTLP endpoints. + aggregator_peer_relation_name: Name of the peers relation of this + charm. This should only be set IFF the charm is an aggregator AND + it has a peer relation with this name. When provided, generic + aggregator rules are used instead of application-level rules. rules: Rules of different types e.g., logql or promql, that the requirer will publish for the provider. """ @@ -185,6 +229,7 @@ def __init__( protocols: Sequence[Literal['http', 'grpc']] | None = None, telemetries: Sequence[Literal['logs', 'metrics', 'traces']] | None = None, *, + aggregator_peer_relation_name: str | None = None, rules: RuleStore | None = None, ): self._charm = charm @@ -196,6 +241,7 @@ def __init__( self._telemetries: list[Literal['logs', 'metrics', 'traces']] = ( list(telemetries) if telemetries is not None else [] ) + self._aggregator_peer_relation_name = aggregator_peer_relation_name self._rules = rules if rules is not None else RuleStore(self._topology) def _filter_endpoints(self, endpoints: list[_OtlpEndpoint]) -> list[_OtlpEndpoint]: @@ -235,6 +281,80 @@ def _favor_modern_endpoints(self, endpoints: list[_OtlpEndpoint]) -> _OtlpEndpoi modern_score: Final = {'grpc': 2, 'http': 1} return max(endpoints, key=lambda e: modern_score.get(e.protocol, 0)) + def _duplicate_rules_per_unit( + self, + alert_rules: OfficialRuleFileFormat, + rule_names_to_duplicate: list[str], + peer_unit_names: set[str], + is_subordinate: bool = False, + ) -> OfficialRuleFileFormat: + """Duplicate alert rule per unit in peer_units list. + + Args: + alert_rules: A dictionary of rules in OfficialRuleFileFormat. + rule_names_to_duplicate: A list of rule names to be duplicated. + peer_unit_names: A set of charm unit names to duplicate rules for. + is_subordinate: A boolean denoting whether the charm duplicating + alert rules is a subordinate or not. If yes, the severity of + the alerts in duplicate_keys needs to be set to critical. + + Returns: + The updated rules with those specified in rule_names_to_duplicate, + duplicated per unit in OfficialRuleFileFormat. + """ + updated_alert_rules = copy.deepcopy(alert_rules) + for group in updated_alert_rules.get('groups', {}): + new_rules: list[SingleRuleFormat] = [] + for rule in group.get('rules', []): + if rule.get('alert', '') not in rule_names_to_duplicate: + new_rules.append(rule) + else: + for juju_unit in sorted(peer_unit_names): + rule_copy = copy.deepcopy(rule) + rule_copy.get('labels', {})['juju_unit'] = juju_unit + rule_copy['expr'] = self._rules.promql.tool.inject_label_matchers( + expression=re.sub(r'%%juju_unit%%,?', '', rule_copy['expr']), + topology={'juju_unit': juju_unit}, + ) + # If the charm is a subordinate, the severity of the alerts need to be + # bumped to critical. + rule_copy.get('labels', {})['severity'] = ( + 'critical' if is_subordinate else 'warning' + ) + new_rules.append(rule_copy) + group['rules'] = new_rules + return updated_alert_rules + + def _inject_generic_rules(self): + """Inject generic rules into the charm's RuleStore.""" + if self._aggregator_peer_relation_name: + if not ( + peer_relations := self._charm.model.get_relation( + self._aggregator_peer_relation_name + ) + ): + logger.warning( + 'Generic aggregator rules were requested, but no peer relation was found. ' + 'Ensure this charm has a peer relation named "%s" to use generic aggregator ' + 'rules.', + self._aggregator_peer_relation_name, + ) + unit_names: set[str] = {self._charm.unit.name} + if peer_relations: + unit_names |= {unit.name for unit in peer_relations.units} + agg_rules = self._duplicate_rules_per_unit( + generic_alert_groups.aggregator_rules, + rule_names_to_duplicate=[HOST_METRICS_MISSING_RULE_NAME], + peer_unit_names=unit_names, + is_subordinate=self._charm.meta.subordinate, + ) + self._rules.add_promql(agg_rules, group_name_prefix=self._topology.identifier) + else: + self._rules.add_promql( + generic_alert_groups.application_rules, + group_name_prefix=self._topology.identifier, + ) + def publish(self): """Triggers programmatically the update of the relation data. @@ -246,10 +366,8 @@ def publish(self): # Only the leader unit can write to app data. return - self._rules.add_promql( - copy.deepcopy(generic_alert_groups.aggregator_rules), - group_name_prefix=self._topology.identifier, - ) + # Add generic rules + self._inject_generic_rules() # Publish to databag databag = _OtlpRequirerAppData.model_validate({ @@ -363,7 +481,9 @@ def rules(self, query_type: Literal['logql', 'promql']) -> dict[str, OfficialRul continue # Get rules for the desired query type - rules_for_type: dict[str, Any] | None = getattr(requirer.rules, query_type, None) + rules_for_type: OfficialRuleFileFormat | None = getattr( + requirer.rules, query_type, None + ) if not rules_for_type: continue diff --git a/interfaces/otlp/src/charmlibs/interfaces/otlp/_version.py b/interfaces/otlp/src/charmlibs/interfaces/otlp/_version.py index 49732b37..a5062d2d 100644 --- a/interfaces/otlp/src/charmlibs/interfaces/otlp/_version.py +++ b/interfaces/otlp/src/charmlibs/interfaces/otlp/_version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.1.0' +__version__ = '0.2.0' diff --git a/interfaces/otlp/tests/unit/conftest.py b/interfaces/otlp/tests/unit/conftest.py index efec06c4..c77153ef 100644 --- a/interfaces/otlp/tests/unit/conftest.py +++ b/interfaces/otlp/tests/unit/conftest.py @@ -19,12 +19,13 @@ import logging import socket from copy import deepcopy -from typing import Final, Literal +from typing import Literal from unittest.mock import patch import ops import pytest from cosl.juju_topology import JujuTopology +from cosl.types import AlertingRuleFormat, OfficialRuleFileFormat, RecordingRuleFormat from ops import testing from ops.charm import CharmBase @@ -33,53 +34,56 @@ logger = logging.getLogger(__name__) +PEERS_ENDPOINT = 'my-peers' LOKI_RULES_DEST_PATH = 'loki_alert_rules' METRICS_RULES_DEST_PATH = 'prometheus_alert_rules' -SINGLE_LOGQL_ALERT: Final = { - 'alert': 'HighLogVolume', - 'expr': 'count_over_time({job=~".+"}[30s]) > 100', - 'labels': {'severity': 'high'}, -} -SINGLE_LOGQL_RECORD: Final = { - 'record': 'log:error_rate:rate5m', - 'expr': 'sum by (service) (rate({job=~".+"} | json | level="error" [5m]))', - 'labels': {'severity': 'high'}, -} -SINGLE_PROMQL_ALERT: Final = { - 'alert': 'Workload Missing', - 'expr': 'up{job=~".+"} == 0', - 'for': '0m', - 'labels': {'severity': 'critical'}, -} -SINGLE_PROMQL_RECORD: Final = { - 'record': 'code:prometheus_http_requests_total:sum', - 'expr': 'sum by (code) (prometheus_http_requests_total{job=~".+"})', - 'labels': {'severity': 'high'}, -} -OFFICIAL_LOGQL_RULES: Final = { - 'groups': [ +SINGLE_LOGQL_ALERT = AlertingRuleFormat( + alert='HighLogVolume', + expr='count_over_time({job=~".+"}[30s]) > 100', + labels={'severity': 'high'}, +) +SINGLE_LOGQL_RECORD = RecordingRuleFormat( + record='log:error_rate:rate5m', + expr='sum by (service) (rate({job=~".+"} | json | level="error" [5m]))', + labels={'severity': 'high'}, +) +SINGLE_PROMQL_ALERT = AlertingRuleFormat( + alert='Workload Missing', + expr='up{job=~".+"} == 0', + for_='0m', + labels={'severity': 'critical'}, +) +SINGLE_PROMQL_RECORD = RecordingRuleFormat( + record='code:prometheus_http_requests_total:sum', + expr='sum by (code) (prometheus_http_requests_total{job=~".+"})', + labels={'severity': 'high'}, +) +OFFICIAL_LOGQL_RULES = OfficialRuleFileFormat( + groups=[ { 'name': 'test_logql', 'rules': [SINGLE_LOGQL_ALERT, SINGLE_LOGQL_RECORD], }, ] -} -OFFICIAL_PROMQL_RULES: Final = { - 'groups': [ +) +OFFICIAL_PROMQL_RULES = OfficialRuleFileFormat( + groups=[ { 'name': 'test_promql', 'rules': [SINGLE_PROMQL_ALERT, SINGLE_PROMQL_RECORD], }, ] -} -ALL_PROTOCOLS: Final[list[Literal['grpc', 'http']]] = ['grpc', 'http'] -ALL_TELEMETRIES: Final[list[Literal['logs', 'metrics', 'traces']]] = ['logs', 'metrics', 'traces'] +) +ALL_PROTOCOLS: list[Literal['grpc', 'http']] = ['grpc', 'http'] +ALL_TELEMETRIES: list[Literal['logs', 'metrics', 'traces']] = ['logs', 'metrics', 'traces'] # --- Tester charms --- class OtlpRequirerCharm(CharmBase): + _aggregator_peer_relation_name: str | None = None + def __init__(self, framework: ops.Framework): super().__init__(framework) self.framework.observe(self.on.update_status, self._publish_rules) @@ -96,7 +100,11 @@ def _publish_rules(self, _: ops.EventBase) -> None: .add_promql(deepcopy(OFFICIAL_PROMQL_RULES)) ) OtlpRequirer( - self, protocols=ALL_PROTOCOLS, telemetries=ALL_TELEMETRIES, rules=rules + self, + protocols=ALL_PROTOCOLS, + telemetries=ALL_TELEMETRIES, + aggregator_peer_relation_name=self._aggregator_peer_relation_name, + rules=rules, ).publish() @@ -121,9 +129,21 @@ def mock_hostname(): @pytest.fixture -def otlp_requirer_ctx() -> testing.Context[OtlpRequirerCharm]: - meta = {'name': 'otlp-requirer', 'requires': {'send-otlp': {'interface': 'otlp'}}} - return testing.Context(OtlpRequirerCharm, meta=meta) +def otlp_requirer_ctx(request: pytest.FixtureRequest) -> testing.Context[OtlpRequirerCharm]: + meta = { + 'name': 'otlp-requirer', + 'requires': {'send-otlp': {'interface': 'otlp'}}, + 'peers': {PEERS_ENDPOINT: {'interface': 'aggregator_peers'}}, + } + # We want to be able to test generic aggregator rules injection and the application rules + # injection case, which is toggled by an aggregator peer relation name input. + generic_aggregator_rules: bool = getattr(request, 'param', False) + charm_cls = type( + 'OtlpRequirerCharm', + (OtlpRequirerCharm,), + {'_aggregator_peer_relation_name': PEERS_ENDPOINT if generic_aggregator_rules else None}, + ) + return testing.Context(charm_cls, meta=meta) @pytest.fixture diff --git a/interfaces/otlp/tests/unit/test_rules.py b/interfaces/otlp/tests/unit/test_rules.py index 67b896c2..0b8638c2 100644 --- a/interfaces/otlp/tests/unit/test_rules.py +++ b/interfaces/otlp/tests/unit/test_rules.py @@ -8,14 +8,21 @@ import ops import pytest +from cosl.rules import HOST_METRICS_MISSING_RULE_NAME from cosl.utils import LZMABase64 from ops import testing -from ops.testing import Model, Relation, State +from ops.testing import Model, PeerRelation, Relation, State from charmlibs.interfaces.otlp._otlp import DEFAULT_PROVIDER_RELATION_NAME as RECEIVE from charmlibs.interfaces.otlp._otlp import DEFAULT_REQUIRER_RELATION_NAME as SEND -from charmlibs.interfaces.otlp._otlp import OtlpProvider, _OtlpRequirerAppData, _RulesModel +from charmlibs.interfaces.otlp._otlp import ( + OtlpProvider, + OtlpRequirer, + _OtlpRequirerAppData, + _RulesModel, +) from conftest import ( + PEERS_ENDPOINT, SINGLE_LOGQL_ALERT, SINGLE_LOGQL_RECORD, SINGLE_PROMQL_ALERT, @@ -64,7 +71,7 @@ def test_rules_compression(otlp_requirer_ctx: testing.Context[ops.CharmBase]): # GIVEN a send-otlp relation state = State(relations=[Relation(SEND)], leader=True) - # WHEN any event executes the reconciler + # WHEN the update_status event is fired state_out = otlp_requirer_ctx.run(otlp_requirer_ctx.on.update_status(), state=state) for relation in list(state_out.relations): rules = relation.local_app_data.get('rules', None) @@ -79,30 +86,111 @@ def test_rules_compression(otlp_requirer_ctx: testing.Context[ops.CharmBase]): assert set(_RulesModel.model_fields.keys()).issubset(decompressed.keys()) -def test_generic_rule_injection(otlp_requirer_ctx: testing.Context[ops.CharmBase]): +@pytest.mark.parametrize('subordinate', [True, False]) +def test_duplicate_rules_per_unit( + otlp_requirer_ctx: testing.Context[ops.CharmBase], subordinate: bool +): + with otlp_requirer_ctx(otlp_requirer_ctx.on.update_status(), state=State(leader=True)) as mgr: + # GIVEN any charm + charm_any = cast('Any', mgr.charm) + # WHEN the OtlpRequirer is initialized + # * generic aggregator rules are desired + # * no peer relation name is provided + result = OtlpRequirer(charm_any)._duplicate_rules_per_unit( + alert_rules={ + 'groups': [ + { + 'name': 'AggregatorHostHealth', + 'rules': [ + { + 'alert': HOST_METRICS_MISSING_RULE_NAME, + 'expr': 'absent(up)', + 'labels': {'severity': 'warning'}, + }, + { + 'alert': 'AggregatorMetricsMissing', + 'expr': 'absent(up)', + 'labels': {'severity': 'critical'}, + }, + ], + } + ] + }, + peer_unit_names={'unit/0', 'unit/1'}, + rule_names_to_duplicate=[HOST_METRICS_MISSING_RULE_NAME], + is_subordinate=subordinate, + ) + # THEN the rules are duplicated per unit, with juju_unit in the expr and labels + groups = result.get('groups', []) + for group in groups: + rules = group.get('rules', []) + severity = 'critical' if subordinate else 'warning' + assert rules == [ + { + 'alert': HOST_METRICS_MISSING_RULE_NAME, + 'expr': 'absent(up{juju_unit="unit/0"})', + 'labels': {'severity': severity, 'juju_unit': 'unit/0'}, + }, + { + 'alert': HOST_METRICS_MISSING_RULE_NAME, + 'expr': 'absent(up{juju_unit="unit/1"})', + 'labels': {'severity': severity, 'juju_unit': 'unit/1'}, + }, + { + 'alert': 'AggregatorMetricsMissing', + 'expr': 'absent(up)', + 'labels': {'severity': 'critical'}, + }, + ] + + +@pytest.mark.parametrize( + 'otlp_requirer_ctx,is_aggregator', + [ + pytest.param(True, True, id='aggregator'), + pytest.param(False, False, id='non-aggregator'), + ], + indirect=['otlp_requirer_ctx'], +) +def test_generic_rule_injection( + otlp_requirer_ctx: testing.Context[ops.CharmBase], is_aggregator: bool +): # GIVEN a send-otlp relation - state = State(relations=[Relation(SEND)], leader=True, model=MODEL) + # * a peers relation + peers = PeerRelation(endpoint=PEERS_ENDPOINT) + state = State(relations=[peers, Relation(SEND)], leader=True, model=MODEL) - # WHEN any event executes the reconciler + # WHEN the update_status event is fired state_out = otlp_requirer_ctx.run(otlp_requirer_ctx.on.update_status(), state=state) for relation in list(state_out.relations): + if relation.endpoint != SEND: + continue + + # THEN if the charm is an aggregator, generic rules are injected into the databag # AND the rules in the databag are decompressed decompressed = _decompress(relation.local_app_data.get('rules')) assert decompressed - logql_groups = decompressed.get('logql', {}).get('groups', []) promql_groups = decompressed.get('promql', {}).get('groups', []) - assert logql_groups assert promql_groups # THEN the generic promql rule is in the databag - assert any('AggregatorHostHealth' in g.get('name') for g in promql_groups) + base_rule_name = f'{MODEL_NAME.replace("-", "_")}_{MODEL_SHORT_UUID}_otlp_requirer' + agg_rule = f'{base_rule_name}_AggregatorHostHealth_rules' + app_rule = f'{base_rule_name}_HostHealth_rules' + promql_group_names = [g.get('name') for g in promql_groups] + if is_aggregator: + assert agg_rule in promql_group_names + assert app_rule not in promql_group_names + else: + assert app_rule in promql_group_names + assert agg_rule not in promql_group_names def test_metadata(otlp_requirer_ctx: testing.Context[ops.CharmBase]): # GIVEN a send-otlp relation state = State(relations=[Relation(SEND)], leader=True, model=MODEL) - # WHEN any event executes the reconciler + # WHEN the update_status event is fired state_out = otlp_requirer_ctx.run(otlp_requirer_ctx.on.update_status(), state=state) for relation in list(state_out.relations): # THEN the requirer adds its own metadata to the databag diff --git a/interfaces/otlp/uv.lock b/interfaces/otlp/uv.lock index 02862b0c..5e844d0d 100644 --- a/interfaces/otlp/uv.lock +++ b/interfaces/otlp/uv.lock @@ -152,7 +152,7 @@ wheels = [ [[package]] name = "cosl" -version = "1.7.0" +version = "1.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ops" }, @@ -161,9 +161,9 @@ dependencies = [ { name = "tenacity" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/86/a1756838edef0bb2e82bbb51d1164dda731334dc0494f3c69d006c7f6429/cosl-1.7.0.tar.gz", hash = "sha256:ab6e1f74a9ddcd8f55fe36b7bedb4c0fe983b5b35776094b31a31e48e63808b6", size = 149830, upload-time = "2026-03-23T17:55:25.872Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/52/88559372957f7e8b672c1f82c699eec3ee3660b9b5198bcdb27232f5e4ad/cosl-1.8.0.tar.gz", hash = "sha256:282214f5ce167a287cee9de0d7865324bb5f7defb7e4c891df3d4c1a101a84a4", size = 149937, upload-time = "2026-03-27T16:39:35.495Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/35/3d/30d2a98fa06c72abb79aceb06f8e6a9848e806d25f86f4f327afa9b13a0b/cosl-1.7.0-py3-none-any.whl", hash = "sha256:a3469c228a0c89418a5d0c9c2c72b520b4f9973fab35ac193b1de9adfac8a4fb", size = 37833, upload-time = "2026-03-23T17:55:24.592Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3b/c73c4bdc8e2acadc05350ffbeca933b2eed5ed9f1f09d3b096b88e23a75a/cosl-1.8.0-py3-none-any.whl", hash = "sha256:f4fef3561fd72187e2067c06a1e7368bdd9865b2a74b9ece105a732eb5438b7e", size = 38045, upload-time = "2026-03-27T16:39:34.395Z" }, ] [[package]]