diff --git a/docs/02_concepts/code/07_webhook.py b/docs/02_concepts/code/07_webhook.py index 76c9153c..0fd15abe 100644 --- a/docs/02_concepts/code/07_webhook.py +++ b/docs/02_concepts/code/07_webhook.py @@ -7,7 +7,7 @@ async def main() -> None: async with Actor: # Create a webhook that will be triggered when the Actor run fails. webhook = Webhook( - event_types=['ACTOR.RUN.FAILED'], # ty: ignore[invalid-argument-type] + event_types=['ACTOR.RUN.FAILED'], request_url='https://example.com/run-failed', ) diff --git a/docs/02_concepts/code/07_webhook_preventing.py b/docs/02_concepts/code/07_webhook_preventing.py index 3ace707b..04c6d5f6 100644 --- a/docs/02_concepts/code/07_webhook_preventing.py +++ b/docs/02_concepts/code/07_webhook_preventing.py @@ -7,7 +7,7 @@ async def main() -> None: async with Actor: # Create a webhook that will be triggered when the Actor run fails. webhook = Webhook( - event_types=['ACTOR.RUN.FAILED'], # ty: ignore[invalid-argument-type] + event_types=['ACTOR.RUN.FAILED'], request_url='https://example.com/run-failed', ) diff --git a/pyproject.toml b/pyproject.toml index 265b7e73..195b0324 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ keywords = [ "scraping", ] dependencies = [ - "apify-client>=2.3.0,<3.0.0", + "apify-client @ git+https://github.com/apify/apify-client-python.git@typed-clients", "apify-shared>=2.0.0,<3.0.0", "crawlee>=1.0.4,<2.0.0", "cachetools>=5.5.0", diff --git a/src/apify/__init__.py b/src/apify/__init__.py index f6495d55..49c085e5 100644 --- a/src/apify/__init__.py +++ b/src/apify/__init__.py @@ -1,6 +1,6 @@ from importlib import metadata -from apify_shared.consts import WebhookEventType +from apify_client._models import WebhookEventType from crawlee import Request from crawlee.events import ( Event, diff --git a/src/apify/_actor.py b/src/apify/_actor.py index fac2ea8b..664947fa 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -885,7 +885,8 @@ async def start( f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, `"RemainingTime"`, or a `timedelta`.' ) - api_result = await client.actor(actor_id).start( + actor_client = client.actor(actor_id) + run = await actor_client.start( run_input=run_input, content_type=content_type, build=build, @@ -895,7 +896,11 @@ async def start( webhooks=serialized_webhooks, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to start Actor with ID "{actor_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def abort( self, @@ -923,13 +928,18 @@ async def abort( self._raise_if_not_initialized() client = self.new_client(token=token) if token else self.apify_client + run_client = client.run(run_id) if status_message: - await client.run(run_id).update(status_message=status_message) + await run_client.update(status_message=status_message) + + run = await run_client.abort(gracefully=gracefully) - api_result = await client.run(run_id).abort(gracefully=gracefully) + if run is None: + raise RuntimeError(f'Failed to abort Actor run with ID "{run_id}".') - return ActorRun.model_validate(api_result) + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def call( self, @@ -1002,7 +1012,8 @@ async def call( f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, `"RemainingTime"`, or a `timedelta`.' ) - api_result = await client.actor(actor_id).call( + actor_client = client.actor(actor_id) + run = await actor_client.call( run_input=run_input, content_type=content_type, build=build, @@ -1013,7 +1024,11 @@ async def call( logger=logger, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def call_task( self, @@ -1075,7 +1090,8 @@ async def call_task( else: raise ValueError(f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, or a `timedelta`.') - api_result = await client.task(task_id).call( + task_client = client.task(task_id) + run = await task_client.call( task_input=task_input, build=build, memory_mbytes=memory_mbytes, @@ -1084,7 +1100,11 @@ async def call_task( wait_secs=int(wait.total_seconds()) if wait is not None else None, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError(f'Failed to call Task with ID "{task_id}".') + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def metamorph( self, @@ -1261,11 +1281,19 @@ async def set_status_message( if not self.configuration.actor_run_id: raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.') - api_result = await self.apify_client.run(self.configuration.actor_run_id).update( - status_message=status_message, is_status_message_terminal=is_terminal + run_client = self.apify_client.run(self.configuration.actor_run_id) + run = await run_client.update( + status_message=status_message, + is_status_message_terminal=is_terminal, ) - return ActorRun.model_validate(api_result) + if run is None: + raise RuntimeError( + f'Failed to set status message for Actor run with ID "{self.configuration.actor_run_id}".' + ) + + run_dict = run.model_dump(by_alias=True) + return ActorRun.model_validate(run_dict) async def create_proxy_configuration( self, diff --git a/src/apify/_charging.py b/src/apify/_charging.py index f8f8ddb2..97040474 100644 --- a/src/apify/_charging.py +++ b/src/apify/_charging.py @@ -351,14 +351,21 @@ async def _fetch_pricing_info(self) -> _FetchedPricingInfoDict: if self._actor_run_id is None: raise RuntimeError('Actor run ID not found even though the Actor is running on Apify') - run = run_validator.validate_python(await self._client.run(self._actor_run_id).get()) + run = await self._client.run(self._actor_run_id).get() + if run is None: raise RuntimeError('Actor run not found') + run_dict = run.model_dump(by_alias=True) + actor_run = run_validator.validate_python(run_dict) + + if actor_run is None: + raise RuntimeError('Actor run not found') + return _FetchedPricingInfoDict( - pricing_info=run.pricing_info, - charged_event_counts=run.charged_event_counts or {}, - max_total_charge_usd=run.options.max_total_charge_usd or Decimal('inf'), + pricing_info=actor_run.pricing_info, + charged_event_counts=actor_run.charged_event_counts or {}, + max_total_charge_usd=actor_run.options.max_total_charge_usd or Decimal('inf'), ) # Local development without environment variables diff --git a/src/apify/_models.py b/src/apify/_models.py index 6dd72dca..46b97765 100644 --- a/src/apify/_models.py +++ b/src/apify/_models.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict, Field -from apify_shared.consts import ActorJobStatus, MetaOrigin, WebhookEventType +from apify_client._models import ActorJobStatus from crawlee._utils.models import timedelta_ms from crawlee._utils.urls import validate_http_url @@ -18,10 +18,10 @@ @docs_group('Actor') class Webhook(BaseModel): - __model_config__ = ConfigDict(populate_by_name=True) + __model_config__ = ConfigDict(populate_by_name=True, use_enum_values=True) event_types: Annotated[ - list[WebhookEventType], + list[str], Field(description='Event types that should trigger the webhook'), ] request_url: Annotated[ @@ -37,9 +37,9 @@ class Webhook(BaseModel): @docs_group('Actor') class ActorRunMeta(BaseModel): - __model_config__ = ConfigDict(populate_by_name=True) + __model_config__ = ConfigDict(populate_by_name=True, use_enum_values=True) - origin: Annotated[MetaOrigin, Field()] + origin: Annotated[str, Field()] @docs_group('Actor') @@ -94,32 +94,82 @@ class ActorRunUsage(BaseModel): @docs_group('Actor') class ActorRun(BaseModel): - __model_config__ = ConfigDict(populate_by_name=True) + """Represents an Actor run and its associated data.""" + + __model_config__ = ConfigDict(populate_by_name=True, use_enum_values=True) id: Annotated[str, Field(alias='id')] + """Unique identifier of the Actor run.""" + act_id: Annotated[str, Field(alias='actId')] + """ID of the Actor that was run.""" + user_id: Annotated[str, Field(alias='userId')] + """ID of the user who started the run.""" + actor_task_id: Annotated[str | None, Field(alias='actorTaskId')] = None + """ID of the Actor task, if the run was started from a task.""" + started_at: Annotated[datetime, Field(alias='startedAt')] + """Time when the Actor run started.""" + finished_at: Annotated[datetime | None, Field(alias='finishedAt')] = None + """Time when the Actor run finished.""" + status: Annotated[ActorJobStatus, Field(alias='status')] + """Current status of the Actor run.""" + status_message: Annotated[str | None, Field(alias='statusMessage')] = None + """Detailed message about the run status.""" + is_status_message_terminal: Annotated[bool | None, Field(alias='isStatusMessageTerminal')] = None + """Whether the status message is terminal (final).""" + meta: Annotated[ActorRunMeta, Field(alias='meta')] + """Metadata about the Actor run.""" + stats: Annotated[ActorRunStats, Field(alias='stats')] + """Statistics of the Actor run.""" + options: Annotated[ActorRunOptions, Field(alias='options')] + """Configuration options for the Actor run.""" + build_id: Annotated[str, Field(alias='buildId')] + """ID of the Actor build used for this run.""" + exit_code: Annotated[int | None, Field(alias='exitCode')] = None + """Exit code of the Actor run process.""" + default_key_value_store_id: Annotated[str, Field(alias='defaultKeyValueStoreId')] + """ID of the default key-value store for this run.""" + default_dataset_id: Annotated[str, Field(alias='defaultDatasetId')] + """ID of the default dataset for this run.""" + default_request_queue_id: Annotated[str, Field(alias='defaultRequestQueueId')] + """ID of the default request queue for this run.""" + build_number: Annotated[str | None, Field(alias='buildNumber')] = None + """Build number of the Actor build used for this run.""" + container_url: Annotated[str, Field(alias='containerUrl')] + """URL of the container running the Actor.""" + is_container_server_ready: Annotated[bool | None, Field(alias='isContainerServerReady')] = None + """Whether the container's HTTP server is ready to accept requests.""" + git_branch_name: Annotated[str | None, Field(alias='gitBranchName')] = None + """Name of the git branch used for the Actor build.""" + usage: Annotated[ActorRunUsage | None, Field(alias='usage')] = None + """Resource usage statistics for the run.""" + usage_total_usd: Annotated[float | None, Field(alias='usageTotalUsd')] = None + """Total cost of the run in USD.""" + usage_usd: Annotated[ActorRunUsage | None, Field(alias='usageUsd')] = None + """Resource usage costs in USD.""" + pricing_info: Annotated[ FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo @@ -128,10 +178,13 @@ class ActorRun(BaseModel): | None, Field(alias='pricingInfo', discriminator='pricing_model'), ] = None + """Pricing information for the Actor.""" + charged_event_counts: Annotated[ dict[str, int] | None, Field(alias='chargedEventCounts'), ] = None + """Count of charged events for pay-per-event pricing model.""" class FreeActorPricingInfo(BaseModel): diff --git a/src/apify/storage_clients/_apify/_alias_resolving.py b/src/apify/storage_clients/_apify/_alias_resolving.py index e357333f..ea4ecaa6 100644 --- a/src/apify/storage_clients/_apify/_alias_resolving.py +++ b/src/apify/storage_clients/_apify/_alias_resolving.py @@ -14,7 +14,7 @@ from collections.abc import Callable from types import TracebackType - from apify_client.clients import ( + from apify_client._resource_clients import ( DatasetClientAsync, DatasetCollectionClientAsync, KeyValueStoreClientAsync, @@ -105,8 +105,8 @@ async def open_by_alias( # Create new unnamed storage and store alias mapping raw_metadata = await collection_client.get_or_create() - await alias_resolver.store_mapping(storage_id=raw_metadata['id']) - return get_resource_client_by_id(raw_metadata['id']) + await alias_resolver.store_mapping(storage_id=raw_metadata.id) + return get_resource_client_by_id(raw_metadata.id) class AliasResolver: diff --git a/src/apify/storage_clients/_apify/_api_client_creation.py b/src/apify/storage_clients/_apify/_api_client_creation.py index 39e2a087..864e1a6d 100644 --- a/src/apify/storage_clients/_apify/_api_client_creation.py +++ b/src/apify/storage_clients/_apify/_api_client_creation.py @@ -8,7 +8,7 @@ from apify.storage_clients._apify._alias_resolving import open_by_alias if TYPE_CHECKING: - from apify_client.clients import DatasetClientAsync, KeyValueStoreClientAsync, RequestQueueClientAsync + from apify_client._resource_clients import DatasetClientAsync, KeyValueStoreClientAsync, RequestQueueClientAsync from apify._configuration import Configuration @@ -137,13 +137,13 @@ def get_resource_client(storage_id: str) -> DatasetClientAsync: # Default storage does not exist. Create a new one. if not raw_metadata: raw_metadata = await collection_client.get_or_create() - resource_client = get_resource_client(raw_metadata['id']) + resource_client = get_resource_client(raw_metadata.id) return resource_client # Open by name. case (None, str(), None, _): raw_metadata = await collection_client.get_or_create(name=name) - return get_resource_client(raw_metadata['id']) + return get_resource_client(raw_metadata.id) # Open by ID. case (None, None, str(), _): diff --git a/src/apify/storage_clients/_apify/_dataset_client.py b/src/apify/storage_clients/_apify/_dataset_client.py index a918bddd..b4e1e619 100644 --- a/src/apify/storage_clients/_apify/_dataset_client.py +++ b/src/apify/storage_clients/_apify/_dataset_client.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator - from apify_client.clients import DatasetClientAsync + from apify_client._resource_clients import DatasetClientAsync from crawlee._types import JsonSerializable from apify import Configuration @@ -65,7 +65,18 @@ def __init__( @override async def get_metadata(self) -> DatasetMetadata: metadata = await self._api_client.get() - return DatasetMetadata.model_validate(metadata) + + if metadata is None: + raise ValueError('Failed to retrieve dataset metadata.') + + return DatasetMetadata( + id=metadata.id, + name=metadata.name, + created_at=metadata.created_at, + modified_at=metadata.modified_at, + accessed_at=metadata.accessed_at, + item_count=int(metadata.item_count), + ) @classmethod async def open( diff --git a/src/apify/storage_clients/_apify/_key_value_store_client.py b/src/apify/storage_clients/_apify/_key_value_store_client.py index b422b464..712a5e78 100644 --- a/src/apify/storage_clients/_apify/_key_value_store_client.py +++ b/src/apify/storage_clients/_apify/_key_value_store_client.py @@ -11,12 +11,12 @@ from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata from ._api_client_creation import create_storage_api_client -from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage +from ._models import ApifyKeyValueStoreMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator - from apify_client.clients import KeyValueStoreClientAsync + from apify_client._resource_clients import KeyValueStoreClientAsync from apify import Configuration @@ -54,7 +54,18 @@ def __init__( @override async def get_metadata(self) -> ApifyKeyValueStoreMetadata: metadata = await self._api_client.get() - return ApifyKeyValueStoreMetadata.model_validate(metadata) + + if metadata is None: + raise ValueError('Failed to retrieve dataset metadata.') + + return ApifyKeyValueStoreMetadata( + id=metadata.id, + name=metadata.name, + created_at=metadata.created_at, + modified_at=metadata.modified_at, + accessed_at=metadata.accessed_at, + url_signing_secret_key=metadata.url_signing_secret_key, + ) @classmethod async def open( @@ -143,14 +154,13 @@ async def iterate_keys( count = 0 while True: - response = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) - list_key_page = KeyValueStoreListKeysPage.model_validate(response) + list_key_page = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) for item in list_key_page.items: # Convert KeyValueStoreKeyInfo to KeyValueStoreRecordMetadata record_metadata = KeyValueStoreRecordMetadata( key=item.key, - size=item.size, + size=int(item.size), content_type='application/octet-stream', # Content type not available from list_keys ) yield record_metadata diff --git a/src/apify/storage_clients/_apify/_request_queue_client.py b/src/apify/storage_clients/_apify/_request_queue_client.py index 9a589ec1..b62e41c6 100644 --- a/src/apify/storage_clients/_apify/_request_queue_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_client.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync from crawlee import Request from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata @@ -77,26 +77,31 @@ async def get_metadata(self) -> ApifyRequestQueueMetadata: Returns: Request queue metadata with accurate counts and timestamps, combining API data with local estimates. """ - response = await self._api_client.get() + metadata = await self._api_client.get() - if response is None: + if metadata is None: raise ValueError('Failed to fetch request queue metadata from the API.') + total_request_count = int(metadata.total_request_count) + handled_request_count = int(metadata.handled_request_count) + pending_request_count = int(metadata.pending_request_count) + # Enhance API response with local estimations to account for propagation delays (API data can be delayed # by a few seconds, while local estimates are immediately accurate). return ApifyRequestQueueMetadata( - id=response['id'], - name=response['name'], - total_request_count=max(response['totalRequestCount'], self._implementation.metadata.total_request_count), - handled_request_count=max( - response['handledRequestCount'], self._implementation.metadata.handled_request_count + id=metadata.id, + name=metadata.name, + total_request_count=max(total_request_count, self._implementation.metadata.total_request_count), + handled_request_count=max(handled_request_count, self._implementation.metadata.handled_request_count), + pending_request_count=pending_request_count, + created_at=min(metadata.created_at, self._implementation.metadata.created_at), + modified_at=max(metadata.modified_at, self._implementation.metadata.modified_at), + accessed_at=max(metadata.accessed_at, self._implementation.metadata.accessed_at), + had_multiple_clients=metadata.had_multiple_clients or self._implementation.metadata.had_multiple_clients, + stats=RequestQueueStats.model_validate( + metadata.stats.model_dump(by_alias=True) if metadata.stats else {}, + by_alias=True, ), - pending_request_count=response['pendingRequestCount'], - created_at=min(response['createdAt'], self._implementation.metadata.created_at), - modified_at=max(response['modifiedAt'], self._implementation.metadata.modified_at), - accessed_at=max(response['accessedAt'], self._implementation.metadata.accessed_at), - had_multiple_clients=response['hadMultipleClients'] or self._implementation.metadata.had_multiple_clients, - stats=RequestQueueStats.model_validate(response['stats'], by_alias=True), ) @classmethod @@ -145,7 +150,7 @@ async def open( raw_metadata = await api_client.get() if raw_metadata is None: raise ValueError('Failed to retrieve request queue metadata from the API.') - metadata = ApifyRequestQueueMetadata.model_validate(raw_metadata) + metadata = ApifyRequestQueueMetadata.model_validate(raw_metadata.model_dump(by_alias=True)) return cls( api_client=api_client, diff --git a/src/apify/storage_clients/_apify/_request_queue_shared_client.py b/src/apify/storage_clients/_apify/_request_queue_shared_client.py index 4a00e8bc..e7a9344e 100644 --- a/src/apify/storage_clients/_apify/_request_queue_shared_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_shared_client.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Coroutine, Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync logger = getLogger(__name__) @@ -121,18 +121,17 @@ async def add_batch_of_requests( if new_requests: # Prepare requests for API by converting to dictionaries. - requests_dict = [ - request.model_dump( - by_alias=True, - ) - for request in new_requests - ] + requests_dict = [request.model_dump(by_alias=True) for request in new_requests] # Send requests to API. - api_response = AddRequestsResponse.model_validate( - await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) + batch_response = await self._api_client.batch_add_requests( + requests=requests_dict, + forefront=forefront, ) + batch_response_dict = batch_response.model_dump(by_alias=True) + api_response = AddRequestsResponse.model_validate(batch_response_dict) + # Add the locally known already present processed requests based on the local cache. api_response.processed_requests.extend(already_present_requests) @@ -312,7 +311,8 @@ async def _get_request_by_id(self, request_id: str) -> Request | None: if response is None: return None - return Request.model_validate(response) + response_dict = response.model_dump(by_alias=True) + return Request.model_validate(response_dict) async def _ensure_head_is_non_empty(self) -> None: """Ensure that the queue head has requests if they are available in the queue.""" @@ -388,7 +388,7 @@ async def _update_request( ) return ProcessedRequest.model_validate( - {'uniqueKey': request.unique_key} | response, + {'uniqueKey': request.unique_key} | response.model_dump(by_alias=True), ) async def _list_head( @@ -431,19 +431,19 @@ async def _list_head( self._should_check_for_forefront_requests = False # Otherwise fetch from API - response = await self._api_client.list_and_lock_head( + locked_queue_head = await self._api_client.list_and_lock_head( lock_secs=int(self._DEFAULT_LOCK_TIME.total_seconds()), limit=limit, ) # Update the queue head cache - self._queue_has_locked_requests = response.get('queueHasLockedRequests', False) + self._queue_has_locked_requests = locked_queue_head.queue_has_locked_requests # Check if there is another client working with the RequestQueue - self.metadata.had_multiple_clients = response.get('hadMultipleClients', False) + self.metadata.had_multiple_clients = locked_queue_head.had_multiple_clients - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data.get('id') + for request_data in locked_queue_head.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id # Skip requests without ID or unique key if not request.unique_key or not request_id: @@ -473,7 +473,8 @@ async def _list_head( # After adding new requests to the forefront, any existing leftover locked request is kept in the end. self._queue_head.append(leftover_id) - return RequestQueueHead.model_validate(response) + list_and_lost_dict = locked_queue_head.model_dump(by_alias=True) + return RequestQueueHead.model_validate(list_and_lost_dict) def _cache_request( self, diff --git a/src/apify/storage_clients/_apify/_request_queue_single_client.py b/src/apify/storage_clients/_apify/_request_queue_single_client.py index 7cc202bb..578196cc 100644 --- a/src/apify/storage_clients/_apify/_request_queue_single_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_single_client.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from apify_client.clients import RequestQueueClientAsync + from apify_client._resource_clients import RequestQueueClientAsync logger = getLogger(__name__) @@ -147,22 +147,20 @@ async def add_batch_of_requests( if new_requests: # Prepare requests for API by converting to dictionaries. - requests_dict = [ - request.model_dump( - by_alias=True, - ) - for request in new_requests - ] + requests_dict = [request.model_dump(by_alias=True) for request in new_requests] # Send requests to API. - api_response = AddRequestsResponse.model_validate( - await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) - ) + batch_response = await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront) + batch_response_dict = batch_response.model_dump(by_alias=True) + api_response = AddRequestsResponse.model_validate(batch_response_dict) + # Add the locally known already present processed requests based on the local cache. api_response.processed_requests.extend(already_present_requests) + # Remove unprocessed requests from the cache for unprocessed_request in api_response.unprocessed_requests: - self._requests_cache.pop(unique_key_to_request_id(unprocessed_request.unique_key), None) + request_id = unique_key_to_request_id(unprocessed_request.unique_key) + self._requests_cache.pop(request_id, None) else: api_response = AddRequestsResponse( @@ -288,16 +286,16 @@ async def _list_head(self) -> None: # Update metadata # Check if there is another client working with the RequestQueue - self.metadata.had_multiple_clients = response.get('hadMultipleClients', False) + self.metadata.had_multiple_clients = response.had_multiple_clients # Should warn once? This might be outside expected context if the other consumers consumes at the same time - if modified_at := response.get('queueModifiedAt'): - self.metadata.modified_at = max(self.metadata.modified_at, modified_at) + if response.queue_modified_at: + self.metadata.modified_at = max(self.metadata.modified_at, response.queue_modified_at) # Update the cached data - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data['id'] + for request_data in response.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id if request_id in self._requests_in_progress: # Ignore requests that are already in progress, we will not process them again. @@ -328,7 +326,8 @@ async def _get_request_by_id(self, id: str) -> Request | None: if response is None: return None - request = Request.model_validate(response) + response_dict = response.model_dump(by_alias=True) + request = Request.model_validate(response_dict) # Updated local caches if id in self._requests_in_progress: @@ -365,7 +364,7 @@ async def _update_request( ) return ProcessedRequest.model_validate( - {'uniqueKey': request.unique_key} | response, + {'uniqueKey': request.unique_key} | response.model_dump(by_alias=True), ) async def _init_caches(self) -> None: @@ -378,9 +377,9 @@ async def _init_caches(self) -> None: Local deduplication is cheaper, it takes 1 API call for whole cache and 1 read operation per request. """ response = await self._api_client.list_requests(limit=10_000) - for request_data in response.get('items', []): - request = Request.model_validate(request_data) - request_id = request_data['id'] + for request_data in response.items: + request = Request.model_validate(request_data.model_dump(by_alias=True)) + request_id = request_data.id if request.was_already_handled: # Cache just id for deduplication diff --git a/tests/integration/README.md b/tests/integration/README.md index 4e07ec51..0e5a6ca7 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -52,7 +52,7 @@ async def test_something( actor = await make_actor(label='something', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' ``` These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. @@ -74,7 +74,7 @@ async def test_something( actor = await make_actor(label='something', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' ``` #### Creating Actor from source files @@ -135,7 +135,7 @@ async def test_something( actor = await make_actor(label='something', source_files=actor_source_files) actor_run = await run_actor(actor) - assert actor_run.status == 'SUCCEEDED' + assert actor_run.status.value == 'SUCCEEDED' ``` #### Asserts @@ -158,5 +158,5 @@ async def test_add_and_fetch_requests( actor = await make_actor(label='rq-test', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' ``` diff --git a/tests/integration/actor/conftest.py b/tests/integration/actor/conftest.py index cba40178..5c540f5d 100644 --- a/tests/integration/actor/conftest.py +++ b/tests/integration/actor/conftest.py @@ -13,7 +13,7 @@ from filelock import FileLock from apify_client import ApifyClient, ApifyClientAsync -from apify_shared.consts import ActorJobStatus, ActorPermissionLevel, ActorSourceType +from apify_client._models import ActorPermissionLevel, VersionSourceType from .._utils import generate_unique_resource_name from apify._models import ActorRun @@ -22,7 +22,7 @@ from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from decimal import Decimal - from apify_client.clients.resource_clients import ActorClientAsync + from apify_client._resource_clients import ActorClientAsync _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' @@ -230,25 +230,25 @@ async def _make_actor( { 'versionNumber': '0.0', 'buildTag': 'latest', - 'sourceType': ActorSourceType.SOURCE_FILES, + 'sourceType': VersionSourceType.SOURCE_FILES.value, 'sourceFiles': source_files_for_api, } ], ) - actor_client = client.actor(created_actor['id']) + actor_client = client.actor(created_actor.id) print(f'Building Actor {actor_name}...') build_result = await actor_client.build(version_number='0.0') - build_client = client.build(build_result['id']) + build_client = client.build(build_result.id) build_client_result = await build_client.wait_for_finish(wait_secs=600) assert build_client_result is not None - assert build_client_result['status'] == ActorJobStatus.SUCCEEDED + assert build_client_result.status.value == 'SUCCEEDED' # We only mark the client for cleanup if the build succeeded, so that if something goes wrong here, # you have a chance to check the error. - actors_for_cleanup.append(created_actor['id']) + actors_for_cleanup.append(created_actor.id) return actor_client @@ -256,17 +256,15 @@ async def _make_actor( # Delete all the generated Actors. for actor_id in actors_for_cleanup: - actor_client = ApifyClient(token=apify_token, api_url=os.getenv(_API_URL_ENV_VAR)).actor(actor_id) - - if (actor := actor_client.get()) is not None: - actor_client.update( - pricing_infos=[ - *actor.get('pricingInfos', []), - { - 'pricingModel': 'FREE', - }, - ] - ) + apify_client = ApifyClient(token=apify_token, api_url=os.getenv(_API_URL_ENV_VAR)) + actor_client = apify_client.actor(actor_id) + actor = actor_client.get() + + if actor is not None and actor.pricing_infos is not None: + # Convert Pydantic models to dicts before mixing with plain dict + existing_pricing_infos = [pi.model_dump(by_alias=True, exclude_none=True) for pi in actor.pricing_infos] + new_pricing_infos = [*existing_pricing_infos, {'pricingModel': 'FREE'}] + actor_client.update(pricing_infos=new_pricing_infos) actor_client.delete() @@ -315,12 +313,14 @@ async def _run_actor( force_permission_level=force_permission_level, ) - assert isinstance(call_result, dict), 'The result of ActorClientAsync.call() is not a dictionary.' - assert 'id' in call_result, 'The result of ActorClientAsync.call() does not contain an ID.' + assert call_result is not None, 'Failed to start Actor run: missing run ID in the response.' + + run_client = apify_client_async.run(call_result.id) + actor_run = await run_client.wait_for_finish(wait_secs=600) - run_client = apify_client_async.run(call_result['id']) - run_result = await run_client.wait_for_finish(wait_secs=600) + assert actor_run is not None, 'Actor run did not finish successfully within the expected time.' - return ActorRun.model_validate(run_result) + actor_run_dict = actor_run.model_dump(by_alias=True) + return ActorRun.model_validate(actor_run_dict) return _run_actor diff --git a/tests/integration/actor/test_actor_api_helpers.py b/tests/integration/actor/test_actor_api_helpers.py index 1a18adb7..45f0a23d 100644 --- a/tests/integration/actor/test_actor_api_helpers.py +++ b/tests/integration/actor/test_actor_api_helpers.py @@ -4,7 +4,7 @@ import json from typing import TYPE_CHECKING -from apify_shared.consts import ActorPermissionLevel +from apify_client._models import ActorPermissionLevel from crawlee._utils.crypto import crypto_random_object_id from .._utils import generate_unique_resource_name @@ -28,7 +28,7 @@ async def main() -> None: actor = await make_actor(label='is-at-home', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_retrieves_env_vars( @@ -52,7 +52,7 @@ async def main() -> None: actor = await make_actor(label='get-env', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_creates_new_client_instance( @@ -76,7 +76,7 @@ async def main() -> None: actor = await make_actor(label='new-client', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None @@ -95,13 +95,13 @@ async def main() -> None: actor = await make_actor(label='set-status-message', main_func=main) run_result_1 = await run_actor(actor) - assert run_result_1.status == 'SUCCEEDED' + assert run_result_1.status.value == 'SUCCEEDED' assert run_result_1.status_message == 'testing-status-message' assert run_result_1.is_status_message_terminal is None run_result_2 = await run_actor(actor, run_input={'is_terminal': True}) - assert run_result_2.status == 'SUCCEEDED' + assert run_result_2.status.value == 'SUCCEEDED' assert run_result_2.status_message == 'testing-status-message' assert run_result_2.is_status_message_terminal is True @@ -129,12 +129,15 @@ async def main_outer() -> None: inner_run_status = await Actor.apify_client.actor(inner_actor_id).last_run().get() assert inner_run_status is not None - assert inner_run_status.get('status') in ['READY', 'RUNNING'] + assert inner_run_status.status.value in {'READY', 'RUNNING'} inner_actor = await make_actor(label='start-inner', main_func=main_inner) outer_actor = await make_actor(label='start-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -142,7 +145,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer.status == 'SUCCEEDED' + assert run_result_outer.status.value == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -172,14 +175,18 @@ async def main_outer() -> None: await Actor.call(inner_actor_id, run_input={'test_value': test_value}) - inner_run_status = await Actor.apify_client.actor(inner_actor_id).last_run().get() - assert inner_run_status is not None - assert inner_run_status.get('status') == 'SUCCEEDED' + run_result_inner = await Actor.apify_client.actor(inner_actor_id).last_run().get() + + assert run_result_inner is not None + assert run_result_inner.status.value == 'SUCCEEDED' inner_actor = await make_actor(label='call-inner', main_func=main_inner) outer_actor = await make_actor(label='call-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -187,7 +194,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer.status == 'SUCCEEDED' + assert run_result_outer.status.value == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -217,14 +224,18 @@ async def main_outer() -> None: await Actor.call_task(inner_task_id) - inner_run_status = await Actor.apify_client.task(inner_task_id).last_run().get() - assert inner_run_status is not None - assert inner_run_status.get('status') == 'SUCCEEDED' + run_result_inner = await Actor.apify_client.task(inner_task_id).last_run().get() + + assert run_result_inner is not None + assert run_result_inner.status.value == 'SUCCEEDED' inner_actor = await make_actor(label='call-task-inner', main_func=main_inner) outer_actor = await make_actor(label='call-task-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() task = await apify_client_async.tasks().create( @@ -235,11 +246,11 @@ async def main_outer() -> None: run_result_outer = await run_actor( outer_actor, - run_input={'test_value': test_value, 'inner_task_id': task['id']}, + run_input={'test_value': test_value, 'inner_task_id': task.id}, force_permission_level=ActorPermissionLevel.FULL_PERMISSIONS, ) - assert run_result_outer.status == 'SUCCEEDED' + assert run_result_outer.status.value == 'SUCCEEDED' await inner_actor.last_run().wait_for_finish(wait_secs=600) @@ -247,7 +258,7 @@ async def main_outer() -> None: assert inner_output_record is not None assert inner_output_record['value'] == f'{test_value}_XXX_{test_value}' - await apify_client_async.task(task['id']).delete() + await apify_client_async.task(task.id).delete() async def test_actor_aborts_another_actor_run( @@ -273,7 +284,7 @@ async def main_outer() -> None: outer_actor = await make_actor(label='abort-outer', main_func=main_outer) run_result_inner = await inner_actor.start(force_permission_level=ActorPermissionLevel.FULL_PERMISSIONS) - inner_run_id = run_result_inner['id'] + inner_run_id = run_result_inner.id run_result_outer = await run_actor( outer_actor, @@ -281,13 +292,18 @@ async def main_outer() -> None: force_permission_level=ActorPermissionLevel.FULL_PERMISSIONS, ) - assert run_result_outer.status == 'SUCCEEDED' + assert run_result_outer.status.value == 'SUCCEEDED' - await inner_actor.last_run().wait_for_finish(wait_secs=600) - inner_actor_last_run_dict = await inner_actor.last_run().get() - inner_actor_last_run = ActorRun.model_validate(inner_actor_last_run_dict) + inner_actor_run_client = inner_actor.last_run() + inner_actor_run = await inner_actor_run_client.wait_for_finish(wait_secs=600) + + if inner_actor_run is None: + raise AssertionError('Failed to get inner actor run after aborting it.') + + inner_actor_run_dict = inner_actor_run.model_dump(by_alias=True) + inner_actor_last_run = ActorRun.model_validate(inner_actor_run_dict) - assert inner_actor_last_run.status == 'ABORTED' + assert inner_actor_last_run.status.value == 'ABORTED' inner_output_record = await inner_actor.last_run().key_value_store().get_record('OUTPUT') assert inner_output_record is None @@ -331,7 +347,10 @@ async def main_outer() -> None: inner_actor = await make_actor(label='metamorph-inner', main_func=main_inner) outer_actor = await make_actor(label='metamorph-outer', main_func=main_outer) - inner_actor_id = (await inner_actor.get() or {})['id'] + inner_actor_get_result = await inner_actor.get() + assert inner_actor_get_result is not None, 'Failed to get inner actor ID' + + inner_actor_id = inner_actor_get_result.id test_value = crypto_random_object_id() run_result_outer = await run_actor( @@ -339,7 +358,7 @@ async def main_outer() -> None: run_input={'test_value': test_value, 'inner_actor_id': inner_actor_id}, ) - assert run_result_outer.status == 'SUCCEEDED' + assert run_result_outer.status.value == 'SUCCEEDED' outer_run_key_value_store = outer_actor.last_run().key_value_store() @@ -377,7 +396,7 @@ async def main() -> None: run_input={'counter_key': 'reboot_counter'}, ) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' not_written_value = await actor.last_run().key_value_store().get_record('THIS_KEY_SHOULD_NOT_BE_WRITTEN') assert not_written_value is None @@ -433,7 +452,7 @@ async def main_client() -> None: await Actor.add_webhook( Webhook( - event_types=[WebhookEventType.ACTOR_RUN_SUCCEEDED], + event_types=[WebhookEventType.ACTOR_RUN_SUCCEEDED.value], request_url=server_actor_container_url, ) ) @@ -444,7 +463,7 @@ async def main_client() -> None: ) server_actor_run = await server_actor.start() - server_actor_container_url = server_actor_run['containerUrl'] + server_actor_container_url = server_actor_run.container_url server_actor_initialized = await server_actor.last_run().key_value_store().get_record('INITIALIZED') while not server_actor_initialized: @@ -456,12 +475,18 @@ async def main_client() -> None: run_input={'server_actor_container_url': server_actor_container_url}, ) - assert ac_run_result.status == 'SUCCEEDED' + assert ac_run_result.status.value == 'SUCCEEDED' + + sa_run_client = server_actor.last_run() + sa_run_client_run = await sa_run_client.wait_for_finish(wait_secs=600) + + if sa_run_client_run is None: + raise AssertionError('Failed to get server actor run after waiting for finish.') - sa_run_result_dict = await server_actor.last_run().wait_for_finish(wait_secs=600) - sa_run_result = ActorRun.model_validate(sa_run_result_dict) + sa_run_client_run_dict = sa_run_client_run.model_dump(by_alias=True) + sa_run_result = ActorRun.model_validate(sa_run_client_run_dict) - assert sa_run_result.status == 'SUCCEEDED' + assert sa_run_result.status.value == 'SUCCEEDED' webhook_body_record = await server_actor.last_run().key_value_store().get_record('WEBHOOK_BODY') assert webhook_body_record is not None diff --git a/tests/integration/actor/test_actor_call_timeouts.py b/tests/integration/actor/test_actor_call_timeouts.py index b6c7ea9f..d3180c86 100644 --- a/tests/integration/actor/test_actor_call_timeouts.py +++ b/tests/integration/actor/test_actor_call_timeouts.py @@ -53,7 +53,7 @@ async def main() -> None: actor = await make_actor(label='inherit-timeout', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_call_inherit_timeout( @@ -102,4 +102,4 @@ async def main() -> None: actor = await make_actor(label='remaining-timeout', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_charge.py b/tests/integration/actor/test_actor_charge.py index d72062bc..a6a46e0a 100644 --- a/tests/integration/actor/test_actor_charge.py +++ b/tests/integration/actor/test_actor_charge.py @@ -6,8 +6,6 @@ import pytest_asyncio -from apify_shared.consts import ActorJobStatus - from apify import Actor from apify._models import ActorRun @@ -15,7 +13,7 @@ from collections.abc import Iterable from apify_client import ApifyClientAsync - from apify_client.clients import ActorClientAsync + from apify_client._resource_clients import ActorClientAsync from .conftest import MakeActorFunction, RunActorFunction @@ -48,13 +46,13 @@ async def main() -> None: }, }, }, - ] + ], ) actor = await actor_client.get() assert actor is not None - return str(actor['id']) + return str(actor.id) @pytest_asyncio.fixture(scope='function', loop_scope='module') @@ -82,11 +80,16 @@ async def test_actor_charge_basic( # Refetch until the platform gets its act together for is_last_attempt, _ in retry_counter(30): await asyncio.sleep(1) - updated_run = await apify_client_async.run(run.id).get() - run = ActorRun.model_validate(updated_run) + + run_client = apify_client_async.run(run.id) + updated_run = await run_client.get() + assert updated_run is not None, 'Updated run should not be None' + + updated_run_dict = updated_run.model_dump(by_alias=True) + run = ActorRun.model_validate(updated_run_dict) try: - assert run.status == ActorJobStatus.SUCCEEDED + assert run.status.value == 'SUCCEEDED' assert run.charged_event_counts == {'foobar': 4} break except AssertionError: @@ -99,17 +102,22 @@ async def test_actor_charge_limit( run_actor: RunActorFunction, apify_client_async: ApifyClientAsync, ) -> None: - run = await run_actor(ppe_actor, max_total_charge_usd=Decimal('0.2')) + run_result = await run_actor(ppe_actor, max_total_charge_usd=Decimal('0.2')) # Refetch until the platform gets its act together for is_last_attempt, _ in retry_counter(30): await asyncio.sleep(1) - updated_run = await apify_client_async.run(run.id).get() - run = ActorRun.model_validate(updated_run) + + run_client = apify_client_async.run(run_result.id) + updated_run = await run_client.get() + assert updated_run is not None, 'Updated run should not be None' + + updated_run_dict = updated_run.model_dump(by_alias=True) + run_result = ActorRun.model_validate(updated_run_dict) try: - assert run.status == ActorJobStatus.SUCCEEDED - assert run.charged_event_counts == {'foobar': 2} + assert run_result.status.value == 'SUCCEEDED' + assert run_result.charged_event_counts == {'foobar': 2} break except AssertionError: if is_last_attempt: diff --git a/tests/integration/actor/test_actor_create_proxy_configuration.py b/tests/integration/actor/test_actor_create_proxy_configuration.py index 9ed60704..7d011dcf 100644 --- a/tests/integration/actor/test_actor_create_proxy_configuration.py +++ b/tests/integration/actor/test_actor_create_proxy_configuration.py @@ -30,7 +30,7 @@ async def main() -> None: actor = await make_actor(label='proxy-configuration', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_create_proxy_configuration_with_groups_and_country( @@ -70,4 +70,4 @@ async def main() -> None: actor = await make_actor(label='proxy-configuration', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_dataset.py b/tests/integration/actor/test_actor_dataset.py index 409df584..2e7c0157 100644 --- a/tests/integration/actor/test_actor_dataset.py +++ b/tests/integration/actor/test_actor_dataset.py @@ -29,7 +29,7 @@ async def main() -> None: actor = await make_actor(label='push-data', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' list_page = await actor.last_run().dataset().list_items() assert list_page.items[0]['id'] == 0 @@ -48,7 +48,7 @@ async def main() -> None: actor = await make_actor(label='push-data-over-9mb', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async for item in actor.last_run().dataset().iterate_items(): assert item['str'] == 'x' * 10000 @@ -71,7 +71,7 @@ async def main() -> None: actor = await make_actor(label='test_dataset_iter_items', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_same_references_in_default_dataset( @@ -87,7 +87,7 @@ async def main() -> None: actor = await make_actor(label='dataset-same-ref-default', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_same_references_in_named_dataset( @@ -115,7 +115,7 @@ async def main() -> None: actor = await make_actor(label='dataset-same-ref-named', main_func=main) run_result = await run_actor(actor, run_input={'datasetName': dataset_name}) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_force_cloud( @@ -139,7 +139,7 @@ async def test_force_cloud( try: dataset_details = await dataset_client.get() assert dataset_details is not None - assert dataset_details.get('name') == dataset_name + assert dataset_details.name == dataset_name dataset_items = await dataset_client.list_items() assert dataset_items.items == [dataset_item] @@ -183,7 +183,7 @@ async def main() -> None: actor = await make_actor(label='dataset-defaults', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_dataset_aliases( @@ -227,4 +227,4 @@ async def main() -> None: actor = await make_actor(label='dataset-aliases', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_events.py b/tests/integration/actor/test_actor_events.py index 4f75630e..0617f0e0 100644 --- a/tests/integration/actor/test_actor_events.py +++ b/tests/integration/actor/test_actor_events.py @@ -60,7 +60,7 @@ async def log_event(data: Any) -> None: actor = await make_actor(label='actor-interval-events', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' dataset_items_page = await actor.last_run().dataset().list_items() persist_state_events = [ @@ -105,4 +105,4 @@ def count_event(data: Any) -> None: actor = await make_actor(label='actor-off-event', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_key_value_store.py b/tests/integration/actor/test_actor_key_value_store.py index 2ed9af29..9273f954 100644 --- a/tests/integration/actor/test_actor_key_value_store.py +++ b/tests/integration/actor/test_actor_key_value_store.py @@ -28,7 +28,7 @@ async def main() -> None: actor = await make_actor(label='kvs-same-ref-default', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_same_references_in_named_kvs( @@ -56,7 +56,7 @@ async def main() -> None: actor = await make_actor(label='kvs-same-ref-named', main_func=main) run_result = await run_actor(actor, run_input={'kvsName': kvs_name}) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_force_cloud( @@ -79,7 +79,7 @@ async def test_force_cloud( try: key_value_store_details = await key_value_store_client.get() assert key_value_store_details is not None - assert key_value_store_details.get('name') == key_value_store_name + assert key_value_store_details.name == key_value_store_name key_value_store_record = await key_value_store_client.get_record('foo') assert key_value_store_record is not None @@ -103,7 +103,7 @@ async def main() -> None: actor = await make_actor(label='actor-get-set-value', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_set_value_in_one_run_and_get_value_in_another( @@ -117,7 +117,7 @@ async def main_set() -> None: actor_set = await make_actor(label='actor-set-value', main_func=main_set) run_result_set = await run_actor(actor_set) - assert run_result_set.status == 'SUCCEEDED' + assert run_result_set.status.value == 'SUCCEEDED' # Externally check if the value is present in key-value store test_record = await actor_set.last_run().key_value_store().get_record('test') @@ -141,9 +141,9 @@ async def main_get() -> None: default_kvs_info = await actor_set.last_run().key_value_store().get() assert default_kvs_info is not None - run_result_get = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info['id']}) + run_result = await run_actor(actor_get, run_input={'kvs-id': default_kvs_info.id}) - assert run_result_get.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_get_input_from_run( @@ -194,7 +194,7 @@ async def main(): }, ) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_generate_public_url_for_kvs_record( @@ -229,7 +229,7 @@ async def main() -> None: actor = await make_actor(label='kvs-get-public-url', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_kvs_defaults( @@ -268,7 +268,7 @@ async def main() -> None: actor = await make_actor(label='kvs-defaults', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_kvs_aliases( @@ -312,4 +312,4 @@ async def main() -> None: actor = await make_actor(label='kvs-aliases', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_lifecycle.py b/tests/integration/actor/test_actor_lifecycle.py index e2f98e6d..77455d3d 100644 --- a/tests/integration/actor/test_actor_lifecycle.py +++ b/tests/integration/actor/test_actor_lifecycle.py @@ -38,7 +38,7 @@ async def main() -> None: actor = await make_actor(label='actor-init', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_init_correctly_in_async_with_block( @@ -55,7 +55,7 @@ async def main() -> None: actor = await make_actor(label='with-actor-init', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_exit_with_different_exit_codes( @@ -73,7 +73,7 @@ async def main() -> None: run_result = await run_actor(actor, run_input={'exit_code': exit_code}) assert run_result.exit_code == exit_code - assert run_result.status == 'FAILED' if exit_code > 0 else 'SUCCEEDED' + assert run_result.status.value == 'FAILED' if exit_code > 0 else 'SUCCEEDED' async def test_actor_fail_with_custom_exit_codes_and_status_messages( @@ -89,18 +89,18 @@ async def main() -> None: run_result = await run_actor(actor) assert run_result.exit_code == 1 - assert run_result.status == 'FAILED' + assert run_result.status.value == 'FAILED' for exit_code in [1, 10, 100]: run_result = await run_actor(actor, run_input={'exit_code': exit_code}) assert run_result.exit_code == exit_code - assert run_result.status == 'FAILED' + assert run_result.status.value == 'FAILED' # Fail with a status message. run_result = await run_actor(actor, run_input={'status_message': 'This is a test message'}) - assert run_result.status == 'FAILED' + assert run_result.status.value == 'FAILED' assert run_result.status_message == 'This is a test message' @@ -116,7 +116,7 @@ async def main() -> None: run_result = await run_actor(actor) assert run_result.exit_code == 91 - assert run_result.status == 'FAILED' + assert run_result.status.value == 'FAILED' async def test_actor_with_crawler_reboot(make_actor: MakeActorFunction, run_actor: RunActorFunction) -> None: @@ -136,8 +136,8 @@ async def main() -> None: requests = ['https://example.com/1', 'https://example.com/2'] run = await Actor.apify_client.run(Actor.configuration.actor_run_id or '').get() - assert run - first_run = run.get('stats', {}).get('rebootCount', 0) == 0 + assert run is not None + first_run = run.stats.reboot_count == 0 @crawler.router.default_handler async def default_handler(context: BasicCrawlingContext) -> None: @@ -159,4 +159,4 @@ async def default_handler(context: BasicCrawlingContext) -> None: actor = await make_actor(label='migration', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_log.py b/tests/integration/actor/test_actor_log.py index 9d80bc90..767539e0 100644 --- a/tests/integration/actor/test_actor_log.py +++ b/tests/integration/actor/test_actor_log.py @@ -43,7 +43,7 @@ async def main() -> None: actor = await make_actor(label='actor-log', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'FAILED' + assert run_result.status.value == 'FAILED' run_log = await actor.last_run().log().get() assert run_log is not None diff --git a/tests/integration/actor/test_actor_request_queue.py b/tests/integration/actor/test_actor_request_queue.py index 1cc4c543..9b211dc9 100644 --- a/tests/integration/actor/test_actor_request_queue.py +++ b/tests/integration/actor/test_actor_request_queue.py @@ -26,7 +26,7 @@ async def main() -> None: actor = await make_actor(label='rq-same-ref-default', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_same_references_in_named_rq( @@ -54,7 +54,7 @@ async def main() -> None: actor = await make_actor(label='rq-same-ref-named', main_func=main) run_result = await run_actor(actor, run_input={'rqName': rq_name}) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_deduplication( @@ -84,8 +84,8 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) # Add same request twice @@ -94,16 +94,21 @@ async def main() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 1 + assert stats_after is not None + assert stats_after.write_count is not None + assert stats_before is not None + assert stats_before.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 1 actor = await make_actor(label='rq-deduplication', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_deduplication_use_extended_unique_key( @@ -133,8 +138,8 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) # Add same request twice @@ -143,16 +148,21 @@ async def main() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 2 + assert stats_after is not None + assert stats_after.write_count is not None + assert stats_before is not None + assert stats_before.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 2 actor = await make_actor(label='rq-deduplication', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_parallel_deduplication( @@ -189,10 +199,13 @@ async def main() -> None: # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=rq.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) + assert stats_before is not None + assert stats_before.write_count is not None + # Add batches of some new and some already present requests in workers async def add_requests_worker() -> None: await rq.add_requests(requests[: next(batch_size)]) @@ -203,16 +216,19 @@ async def add_requests_worker() -> None: await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == len(requests) + assert stats_after is not None + assert stats_after.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == len(requests) actor = await make_actor(label='rq-parallel-deduplication', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_had_multiple_clients_platform( @@ -239,7 +255,7 @@ async def main() -> None: actor = await make_actor(label='rq-had-multiple-clients', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_not_had_multiple_clients_platform( @@ -260,7 +276,7 @@ async def main() -> None: actor = await make_actor(label='rq-not-had-multiple-clients', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_request_queue_not_had_multiple_clients_platform_resurrection( @@ -283,16 +299,23 @@ async def main() -> None: actor = await make_actor(label='rq-clients-resurrection', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' # Resurrect the run, the RequestQueue should still use same client key and thus not have multiple clients. run_client = apify_client_async.run(run_id=run_result.id) # Redirect logs even from the resurrected run streamed_log = await run_client.get_streamed_log(from_start=False) await run_client.resurrect() + async with streamed_log: - run_result = ActorRun.model_validate(await run_client.wait_for_finish(wait_secs=600)) - assert run_result.status == 'SUCCEEDED' + run = await run_client.wait_for_finish(wait_secs=600) + + if run is None: + raise AssertionError('Failed to get resurrected run.') + + run_dict = run.model_dump(by_alias=True) + run_result = ActorRun.model_validate(run_dict) + assert run_result.status.value == 'SUCCEEDED' async def test_rq_defaults( @@ -334,7 +357,7 @@ async def main() -> None: actor = await make_actor(label='rq-defaults', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_rq_aliases( @@ -382,7 +405,7 @@ async def main() -> None: actor = await make_actor(label='rq-aliases', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_concurrent_processing_simulation( @@ -483,7 +506,7 @@ async def worker() -> int: actor = await make_actor(label='rq-concurrent-test', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_rq_isolation( @@ -534,4 +557,4 @@ async def main() -> None: actor = await make_actor(label='rq-isolation-test', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_actor_scrapy.py b/tests/integration/actor/test_actor_scrapy.py index b4dd5fee..202789b5 100644 --- a/tests/integration/actor/test_actor_scrapy.py +++ b/tests/integration/actor/test_actor_scrapy.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from apify_shared.consts import ActorPermissionLevel +from apify_client._models import ActorPermissionLevel if TYPE_CHECKING: from .conftest import MakeActorFunction, RunActorFunction @@ -40,7 +40,7 @@ async def test_actor_scrapy_title_spider( force_permission_level=ActorPermissionLevel.FULL_PERMISSIONS, ) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' items = await actor.last_run().dataset().list_items() diff --git a/tests/integration/actor/test_apify_storages.py b/tests/integration/actor/test_apify_storages.py index f3f3696a..4082727c 100644 --- a/tests/integration/actor/test_apify_storages.py +++ b/tests/integration/actor/test_apify_storages.py @@ -25,4 +25,4 @@ async def main() -> None: actor = await make_actor(label='explicit_storage_init', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_crawlers_with_storages.py b/tests/integration/actor/test_crawlers_with_storages.py index cd0d6941..efa0ec10 100644 --- a/tests/integration/actor/test_crawlers_with_storages.py +++ b/tests/integration/actor/test_crawlers_with_storages.py @@ -38,7 +38,7 @@ async def default_handler(context: ParselCrawlingContext) -> None: actor = await make_actor(label='crawler-max-depth', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_on_platform_max_requests_per_crawl( @@ -74,7 +74,7 @@ async def default_handler(context: ParselCrawlingContext) -> None: actor = await make_actor(label='crawler-max-requests', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' async def test_actor_on_platform_max_request_retries( @@ -110,4 +110,4 @@ async def default_handler(_: ParselCrawlingContext) -> None: actor = await make_actor(label='crawler-max-retries', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' diff --git a/tests/integration/actor/test_fixtures.py b/tests/integration/actor/test_fixtures.py index 865effa9..d6735b3e 100644 --- a/tests/integration/actor/test_fixtures.py +++ b/tests/integration/actor/test_fixtures.py @@ -28,7 +28,7 @@ async def main() -> None: actor = await make_actor(label='make-actor-main-func', main_func=main) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') @@ -52,7 +52,7 @@ async def main(): actor = await make_actor(label='make-actor-main-py', main_py=main_py_source) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') @@ -86,7 +86,7 @@ async def main(): actor = await make_actor(label='make-actor-source-files', source_files=actor_source_files) run_result = await run_actor(actor) - assert run_result.status == 'SUCCEEDED' + assert run_result.status.value == 'SUCCEEDED' output_record = await actor.last_run().key_value_store().get_record('OUTPUT') assert output_record is not None diff --git a/tests/integration/apify_api/test_request_queue.py b/tests/integration/apify_api/test_request_queue.py index e90c1600..41adf2da 100644 --- a/tests/integration/apify_api/test_request_queue.py +++ b/tests/integration/apify_api/test_request_queue.py @@ -8,6 +8,7 @@ import pytest +from apify_client._models import BatchOperationResult, UnprocessedRequest from apify_shared.consts import ApifyEnvVars from crawlee import service_locator from crawlee.crawlers import BasicCrawler @@ -930,7 +931,7 @@ async def test_request_queue_had_multiple_clients( # Check that it is correctly in the API api_response = await api_client.get() assert api_response - assert api_response['hadMultipleClients'] is True + assert api_response.had_multiple_clients is True async def test_request_queue_not_had_multiple_clients( @@ -949,7 +950,7 @@ async def test_request_queue_not_had_multiple_clients( api_client = apify_client_async.request_queue(request_queue_id=rq.id) api_response = await api_client.get() assert api_response - assert api_response['hadMultipleClients'] is False + assert api_response.had_multiple_clients is False async def test_request_queue_simple_and_full_at_the_same_time( @@ -1122,11 +1123,11 @@ async def test_force_cloud( request_queue_details = await request_queue_client.get() assert request_queue_details is not None - assert request_queue_details.get('name') == request_queue_apify.name + assert request_queue_details.name == request_queue_apify.name request_queue_request = await request_queue_client.get_request(request_info.id) assert request_queue_request is not None - assert request_queue_request['url'] == 'http://example.com' + assert request_queue_request.url == 'http://example.com' async def test_request_queue_is_finished( @@ -1161,22 +1162,31 @@ async def test_request_queue_deduplication_unprocessed_requests( # Get raw client, because stats are not exposed in `RequestQueue` class, but are available in raw client rq_client = Actor.apify_client.request_queue(request_queue_id=request_queue_apify.id) _rq = await rq_client.get() - assert _rq - stats_before = _rq.get('stats', {}) + assert _rq is not None + stats_before = _rq.stats Actor.log.info(stats_before) - def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> dict[str, list[dict]]: + assert stats_before is not None + assert stats_before.write_count is not None + + def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> BatchOperationResult: """Simulate API returning unprocessed requests.""" - return { - 'processedRequests': [], - 'unprocessedRequests': [ - {'url': request['url'], 'uniqueKey': request['uniqueKey'], 'method': request['method']} - for request in requests - ], - } + unprocessed_requests = [ + UnprocessedRequest.model_construct( + url=request['url'], + unique_key=request['uniqueKey'], + method=request['method'], + ) + for request in requests + ] + + return BatchOperationResult.model_construct( + processed_requests=[], + unprocessed_requests=unprocessed_requests, + ) with mock.patch( - 'apify_client.clients.resource_clients.request_queue.RequestQueueClientAsync.batch_add_requests', + 'apify_client._resource_clients.request_queue.RequestQueueClientAsync.batch_add_requests', side_effect=return_unprocessed_requests, ): # Simulate failed API call for adding requests. Request was not processed and should not be cached. @@ -1187,8 +1197,11 @@ def return_unprocessed_requests(requests: list[dict], *_: Any, **__: Any) -> dic await asyncio.sleep(10) # Wait to be sure that metadata are updated _rq = await rq_client.get() - assert _rq - stats_after = _rq.get('stats', {}) + assert _rq is not None + stats_after = _rq.stats Actor.log.info(stats_after) - assert (stats_after['writeCount'] - stats_before['writeCount']) == 1 + assert stats_after is not None + assert stats_after.write_count is not None + + assert (stats_after.write_count - stats_before.write_count) == 1 diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index f71cd44c..3796b8f4 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -5,7 +5,8 @@ import pytest from apify_client import ApifyClientAsync -from apify_shared.consts import ApifyEnvVars, WebhookEventType +from apify_client._models import Run, WebhookEventType +from apify_shared.consts import ApifyEnvVars from apify import Actor, Webhook from apify._actor import _ActorType @@ -15,45 +16,48 @@ @pytest.fixture -def fake_actor_run() -> dict: - return { - 'id': 'asdfasdf', - 'buildId': '3ads35', - 'buildNumber': '3.4.5', - 'actId': 'actor_id', - 'actorId': 'actor_id', - 'userId': 'user_id', - 'startedAt': '2024-08-08 12:12:44', - 'status': 'RUNNING', - 'meta': {'origin': 'API'}, - 'containerUrl': 'http://0.0.0.0:3333', - 'defaultDatasetId': 'dhasdrfughaerguoi', - 'defaultKeyValueStoreId': 'asjkldhguiofg', - 'defaultRequestQueueId': 'lkjgklserjghios', - 'stats': { - 'inputBodyLen': 0, - 'restartCount': 0, - 'resurrectCount': 0, - 'memAvgBytes': 0, - 'memMaxBytes': 0, - 'memCurrentBytes': 0, - 'cpuAvgUsage': 0, - 'cpuMaxUsage': 0, - 'cpuCurrentUsage': 0, - 'netRxBytes': 0, - 'netTxBytes': 0, - 'durationMillis': 3333, - 'runTimeSecs': 33, - 'metamorph': 0, - 'computeUnits': 4.33, - }, - 'options': { - 'build': '', - 'timeoutSecs': 44, - 'memoryMbytes': 4096, - 'diskMbytes': 16384, - }, - } +def fake_actor_run() -> Run: + return Run.model_validate( + { + 'id': 'asdfasdf', + 'buildId': '3ads35', + 'buildNumber': '3.4.5', + 'actId': 'actor_id', + 'actorId': 'actor_id', + 'userId': 'user_id', + 'startedAt': '2024-08-08T12:12:44Z', + 'status': 'RUNNING', + 'meta': {'origin': 'API'}, + 'containerUrl': 'http://0.0.0.0:3333', + 'defaultDatasetId': 'dhasdrfughaerguoi', + 'defaultKeyValueStoreId': 'asjkldhguiofg', + 'defaultRequestQueueId': 'lkjgklserjghios', + 'generalAccess': 'RESTRICTED', + 'stats': { + 'inputBodyLen': 0, + 'restartCount': 0, + 'resurrectCount': 0, + 'memAvgBytes': 0, + 'memMaxBytes': 0, + 'memCurrentBytes': 0, + 'cpuAvgUsage': 0, + 'cpuMaxUsage': 0, + 'cpuCurrentUsage': 0, + 'netRxBytes': 0, + 'netTxBytes': 0, + 'durationMillis': 3333, + 'runTimeSecs': 33, + 'metamorph': 0, + 'computeUnits': 4.33, + }, + 'options': { + 'build': '', + 'timeoutSecs': 44, + 'memoryMbytes': 4096, + 'diskMbytes': 16384, + }, + } + ) async def test_new_client_config_creation(monkeypatch: pytest.MonkeyPatch) -> None: @@ -74,7 +78,7 @@ async def test_new_client_config_creation(monkeypatch: pytest.MonkeyPatch) -> No await my_actor.exit() -async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('actor', 'call', return_value=fake_actor_run) actor_id = 'some-actor-id' @@ -86,7 +90,7 @@ async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, f assert apify_client_async_patcher.calls['actor']['call'][0][0][0].resource_id == actor_id -async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('task', 'call', return_value=fake_actor_run) task_id = 'some-task-id' @@ -97,7 +101,7 @@ async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatch assert apify_client_async_patcher.calls['task']['call'][0][0][0].resource_id == task_id -async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) actor_id = 'some-id' @@ -108,7 +112,7 @@ async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id -async def test_abort_actor_run(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: dict) -> None: +async def test_abort_actor_run(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: apify_client_async_patcher.patch('run', 'abort', return_value=fake_actor_run) run_id = 'some-run-id' @@ -146,7 +150,7 @@ async def test_add_webhook_fails_locally(caplog: pytest.LogCaptureFixture) -> No caplog.set_level('WARNING') async with Actor: await Actor.add_webhook( - Webhook(event_types=[WebhookEventType.ACTOR_BUILD_ABORTED], request_url='https://example.com') + Webhook(event_types=[WebhookEventType.ACTOR_BUILD_ABORTED.value], request_url='https://example.com') ) assert len(caplog.records) == 1 diff --git a/tests/unit/actor/test_actor_lifecycle.py b/tests/unit/actor/test_actor_lifecycle.py index 5f03c788..850ac075 100644 --- a/tests/unit/actor/test_actor_lifecycle.py +++ b/tests/unit/actor/test_actor_lifecycle.py @@ -12,6 +12,7 @@ import pytest import websockets.asyncio.server +from apify_client._models import Run from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars from crawlee.events._types import Event, EventPersistStateData @@ -238,31 +239,34 @@ async def handler(websocket: websockets.asyncio.server.ServerConnection) -> None mock_run_client = Mock() mock_run_client.run.return_value.get = AsyncMock( - side_effect=lambda: { - 'id': 'asdf', - 'actId': 'asdf', - 'userId': 'adsf', - 'startedAt': datetime.now(timezone.utc), - 'status': 'RUNNING', - 'meta': {'origin': 'DEVELOPMENT'}, - 'stats': { - 'inputBodyLen': 99, - 'restartCount': 0, - 'resurrectCount': 0, - 'computeUnits': 1, - }, - 'options': { - 'build': 'asdf', - 'timeoutSecs': 4, - 'memoryMbytes': 1024, - 'diskMbytes': 1024, - }, - 'buildId': 'hjkl', - 'defaultDatasetId': 'hjkl', - 'defaultKeyValueStoreId': 'hjkl', - 'defaultRequestQueueId': 'hjkl', - 'containerUrl': 'https://hjkl', - } + side_effect=lambda: Run.model_validate( + { + 'id': 'asdf', + 'actId': 'asdf', + 'userId': 'adsf', + 'startedAt': datetime.now(timezone.utc).isoformat(), + 'status': 'RUNNING', + 'meta': {'origin': 'DEVELOPMENT'}, + 'buildId': 'hjkl', + 'defaultDatasetId': 'hjkl', + 'defaultKeyValueStoreId': 'hjkl', + 'defaultRequestQueueId': 'hjkl', + 'containerUrl': 'https://hjkl', + 'buildNumber': '0.0.1', + 'generalAccess': 'RESTRICTED', + 'stats': { + 'restartCount': 0, + 'resurrectCount': 0, + 'computeUnits': 1, + }, + 'options': { + 'build': 'asdf', + 'timeoutSecs': 4, + 'memoryMbytes': 1024, + 'diskMbytes': 1024, + }, + } + ) ) with mock.patch.object(Actor, 'new_client', return_value=mock_run_client): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3f792ad8..94f1f5f2 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -62,6 +62,7 @@ def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callabl def _prepare_test_env() -> None: if hasattr(apify._actor.Actor, '__wrapped__'): delattr(apify._actor.Actor, '__wrapped__') + apify._actor.Actor._is_initialized = False # Set the environment variable for the local storage directory to the temporary path. diff --git a/uv.lock b/uv.lock index 7a1f9902..3f51b0a9 100644 --- a/uv.lock +++ b/uv.lock @@ -72,7 +72,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "apify-client", specifier = ">=2.3.0,<3.0.0" }, + { name = "apify-client", git = "https://github.com/apify/apify-client-python.git?rev=typed-clients" }, { name = "apify-shared", specifier = ">=2.0.0,<3.0.0" }, { name = "cachetools", specifier = ">=5.5.0" }, { name = "crawlee", specifier = ">=1.0.4,<2.0.0" }, @@ -111,18 +111,14 @@ dev = [ [[package]] name = "apify-client" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } +version = "2.4.1" +source = { git = "https://github.com/apify/apify-client-python.git?rev=typed-clients#ab472942a5f84c501dfb30839387e24a2b378db0" } dependencies = [ { name = "apify-shared" }, { name = "colorama" }, { name = "impit" }, { name = "more-itertools" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/24/6a/82e2d61641508e2a8a0509e78d1641273df901683e7108afc71b078c8488/apify_client-2.4.0.tar.gz", hash = "sha256:efcad708f9091f774f180ced18e2aaaec3b45effcc19b933d2fa0b3059b8a001", size = 368665, upload-time = "2026-01-09T10:33:25.756Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/65/71dd2515b799bded5a767006423a11ebac27f41193372e751672f4c12516/apify_client-2.4.0-py3-none-any.whl", hash = "sha256:073109fa136fd978471eff62bf30eda9a51557a5ee383fe762c89430cee27c30", size = 86156, upload-time = "2026-01-09T10:33:24.524Z" }, -] [[package]] name = "apify-shared"