From 34552d6ff1fcf643f5088de3c7a1552f15539482 Mon Sep 17 00:00:00 2001 From: Willi Date: Mon, 8 Jul 2024 15:29:35 +0530 Subject: [PATCH 1/6] rest_api: Renames JSONResponsePaginator to JSONLinkPaginator. Bumps dlt core dependency to 0.5.0 to import the renamed class. --- sources/rest_api/README.md | 12 ++++++------ sources/rest_api/__init__.py | 2 +- sources/rest_api/config_setup.py | 4 ++-- sources/rest_api/requirements.txt | 2 +- sources/rest_api/typing.py | 9 ++++----- sources/rest_api_pipeline.py | 2 +- tests/rest_api/source_configs.py | 8 ++++---- tests/rest_api/test_configurations.py | 4 ++-- tests/rest_api/test_rest_api_source_offline.py | 2 +- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/sources/rest_api/README.md b/sources/rest_api/README.md index 2ae17bbab..d2878d062 100644 --- a/sources/rest_api/README.md +++ b/sources/rest_api/README.md @@ -130,14 +130,14 @@ Possible paginators are: | Paginator | String Alias | Note | | --------- | ------------ | ---- | | BasePaginator | | | -| HeaderLinkPaginator | `header_links` | | -| JSONResponsePaginator | `json_links` | The pagination metainformation is in a node of the JSON response (see example below) | +| HeaderLinkPaginator | `header_link` | | +| JSONLinkPaginator | `json_link` | The pagination metainformation is in a node of the JSON response (see example below) | | SinglePagePaginator | `single_page` | The response will be interpreted as a single-page response, ignoring possible pagination metadata | -Usage example of the `JSONResponsePaginator`, for a response with the URL of the next page located at `paging.next`: +Usage example of the `JSONLinkPaginator`, for a response with the URL of the next page located at `paging.next`: ```python -"paginator": JSONResponsePaginator( - next_key=["paging", "next"] +"paginator": JSONLinkPaginator( + next_url_path="paging.next"] ) ``` @@ -215,4 +215,4 @@ Resources with the name different from the endpoint string will be: ``` In case you need to have a resource with a name different from the table created, you can pass the property `table_name` too. -For the other properties, see the [resource_defaults](#resource_defaults) above. \ No newline at end of file +For the other properties, see the [resource_defaults](#resource_defaults) above. diff --git a/sources/rest_api/__init__.py b/sources/rest_api/__init__.py index e2104b307..9cf70f76b 100644 --- a/sources/rest_api/__init__.py +++ b/sources/rest_api/__init__.py @@ -79,7 +79,7 @@ def rest_api_source( pokemon_source = rest_api_source({ "client": { "base_url": "https://pokeapi.co/api/v2/", - "paginator": "json_response", + "paginator": "json_link", }, "endpoints": { "pokemon": { diff --git a/sources/rest_api/config_setup.py b/sources/rest_api/config_setup.py index a561919d3..954447229 100644 --- a/sources/rest_api/config_setup.py +++ b/sources/rest_api/config_setup.py @@ -29,7 +29,7 @@ BasePaginator, SinglePagePaginator, HeaderLinkPaginator, - JSONResponsePaginator, + JSONLinkPaginator, JSONResponseCursorPaginator, OffsetPaginator, PageNumberPaginator, @@ -60,7 +60,7 @@ PAGINATOR_MAP: Dict[PaginatorType, Type[BasePaginator]] = { - "json_response": JSONResponsePaginator, + "json_link": JSONLinkPaginator, "header_link": HeaderLinkPaginator, "auto": None, "single_page": SinglePagePaginator, diff --git a/sources/rest_api/requirements.txt b/sources/rest_api/requirements.txt index 7f5d71685..819f778f8 100644 --- a/sources/rest_api/requirements.txt +++ b/sources/rest_api/requirements.txt @@ -1 +1 @@ -dlt>=0.4.11 \ No newline at end of file +dlt>=0.5.0 diff --git a/sources/rest_api/typing.py b/sources/rest_api/typing.py index b70d5a0b7..7b027d7ca 100644 --- a/sources/rest_api/typing.py +++ b/sources/rest_api/typing.py @@ -30,7 +30,6 @@ from dlt.sources.helpers.rest_client.paginators import ( SinglePagePaginator, HeaderLinkPaginator, - JSONResponsePaginator, JSONResponseCursorPaginator, OffsetPaginator, PageNumberPaginator, @@ -43,7 +42,7 @@ ) PaginatorType = Literal[ - "json_response", + "json_link", "header_link", "auto", "single_page", @@ -84,7 +83,7 @@ class HeaderLinkPaginatorConfig(PaginatorTypeConfig, total=False): links_next_key: Optional[str] -class JSONResponsePaginatorConfig(PaginatorTypeConfig, total=False): +class JSONLinkPaginatorConfig(PaginatorTypeConfig, total=False): """Locates the next page URL within the JSON response body. The key containing the URL can be specified using a JSON path.""" @@ -104,12 +103,12 @@ class JSONResponseCursorPaginatorConfig(PaginatorTypeConfig, total=False): PageNumberPaginatorConfig, OffsetPaginatorConfig, HeaderLinkPaginatorConfig, - JSONResponsePaginatorConfig, + JSONLinkPaginatorConfig, JSONResponseCursorPaginatorConfig, BasePaginator, SinglePagePaginator, HeaderLinkPaginator, - JSONResponsePaginator, + JSONLinkPaginator, JSONResponseCursorPaginator, OffsetPaginator, PageNumberPaginator, diff --git a/sources/rest_api_pipeline.py b/sources/rest_api_pipeline.py index 924fc69f1..2ca5dcfbb 100644 --- a/sources/rest_api_pipeline.py +++ b/sources/rest_api_pipeline.py @@ -116,7 +116,7 @@ def load_pokemon() -> None: "client": { "base_url": "https://pokeapi.co/api/v2/", # If you leave out the paginator, it will be inferred from the API: - # paginator: "json_response", + # paginator: "json_link", }, "resource_defaults": { "endpoint": { diff --git a/tests/rest_api/source_configs.py b/tests/rest_api/source_configs.py index 8d3cde3e6..424dd04e0 100644 --- a/tests/rest_api/source_configs.py +++ b/tests/rest_api/source_configs.py @@ -137,7 +137,7 @@ class CustomOAuthAuth(OAuth2AuthBase): "params": { "limit": 100, }, - "paginator": "json_response", + "paginator": "json_link", }, }, ], @@ -217,7 +217,7 @@ class CustomOAuthAuth(OAuth2AuthBase): "initial_value": "2024-01-25T11:21:28Z", }, }, - "paginator": "json_response", + "paginator": "json_link", }, }, ], @@ -232,7 +232,7 @@ class CustomOAuthAuth(OAuth2AuthBase): "params": { "limit": 100, }, - "paginator": "json_response", + "paginator": "json_link", "incremental": { "start_param": "since", "end_param": "until", @@ -313,7 +313,7 @@ class CustomOAuthAuth(OAuth2AuthBase): {"type": "page_number", "page": 10, "base_page": 1, "total_path": "response.pages"}, {"type": "offset", "limit": 100, "maximum_offset": 1000}, {"type": "header_link", "links_next_key": "next_page"}, - {"type": "json_response", "next_url_path": "response.nex_page_link"}, + {"type": "json_link", "next_url_path": "response.nex_page_link"}, {"type": "cursor", "cursor_param": "cursor"}, ] diff --git a/tests/rest_api/test_configurations.py b/tests/rest_api/test_configurations.py index 122639ce1..a69a79244 100644 --- a/tests/rest_api/test_configurations.py +++ b/tests/rest_api/test_configurations.py @@ -51,7 +51,7 @@ ) from dlt.sources.helpers.rest_client.paginators import ( HeaderLinkPaginator, - JSONResponsePaginator, + JSONLinkPaginator, JSONResponseCursorPaginator, OffsetPaginator, PageNumberPaginator, @@ -124,7 +124,7 @@ def test_paginator_type_configs(paginator_type_config: PaginatorTypeConfig) -> N assert paginator.limit_param == "limit" assert paginator.total_path == compile_path("total") assert paginator.maximum_value == 1000 - if isinstance(paginator, JSONResponsePaginator): + if isinstance(paginator, JSONLinkPaginator): assert paginator.next_url_path == compile_path("response.nex_page_link") if isinstance(paginator, JSONResponseCursorPaginator): assert paginator.cursor_path == compile_path("cursors.next") diff --git a/tests/rest_api/test_rest_api_source_offline.py b/tests/rest_api/test_rest_api_source_offline.py index b30b7b092..cdc46807a 100644 --- a/tests/rest_api/test_rest_api_source_offline.py +++ b/tests/rest_api/test_rest_api_source_offline.py @@ -210,7 +210,7 @@ def test_posts_under_results_key(mock_api_server): "endpoint": { "path": "posts_under_a_different_key", "data_selector": "many-results", - "paginator": "json_response", + "paginator": "json_link", }, }, ], From f04de62ada6bddde78f75a146cbc52c3f3f29f8f Mon Sep 17 00:00:00 2001 From: Willi Date: Mon, 8 Jul 2024 22:16:13 +0530 Subject: [PATCH 2/6] import JSONLinkPaginator so that it can be referenced in PaginatorConfig --- sources/rest_api/typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sources/rest_api/typing.py b/sources/rest_api/typing.py index 7b027d7ca..1157bd3fa 100644 --- a/sources/rest_api/typing.py +++ b/sources/rest_api/typing.py @@ -30,6 +30,7 @@ from dlt.sources.helpers.rest_client.paginators import ( SinglePagePaginator, HeaderLinkPaginator, + JSONLinkPaginator, JSONResponseCursorPaginator, OffsetPaginator, PageNumberPaginator, From 9af18362016552379b8ae245d1a8816fec31c000 Mon Sep 17 00:00:00 2001 From: Willi Date: Mon, 8 Jul 2024 22:45:42 +0530 Subject: [PATCH 3/6] sets dlt core dependency. Don't merge this! --- poetry.lock | 33 +++++++++++-------------------- pyproject.toml | 4 ++-- sources/rest_api/requirements.txt | 2 +- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 77063db77..051984990 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand. [[package]] name = "adlfs" @@ -1071,17 +1071,18 @@ files = [ {file = "dlt-0.5.1-py3-none-any.whl", hash = "sha256:19d7920816fadd049a1a92c4ecc4e740bf1cd5f6484fce535715f8c214a835ce"}, {file = "dlt-0.5.1.tar.gz", hash = "sha256:dfa11e498feec3aca0022541850b6fa7dd3fddce4c2e6fef195530638269fec4"}, ] +develop = false [package.dependencies] astunparse = ">=1.6.3" click = ">=7.1" duckdb = {version = ">=0.6.1,<0.11", optional = true, markers = "extra == \"duckdb\" or extra == \"motherduck\""} fsspec = ">=2022.4.0" -gcsfs = {version = ">=2022.4.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\" or extra == \"gs\" or extra == \"clickhouse\""} +gcsfs = {version = ">=2022.4.0", optional = true} gitpython = ">=3.1.29" giturlparse = ">=0.10.0" -google-cloud-bigquery = {version = ">=2.26.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\""} -grpcio = {version = ">=1.50.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\""} +google-cloud-bigquery = {version = ">=2.26.0", optional = true} +grpcio = {version = ">=1.50.0", optional = true} hexbytes = ">=0.2.2" humanize = ">=4.4.0" jsonpath-ng = ">=1.5.3" @@ -1131,6 +1132,12 @@ snowflake = ["snowflake-connector-python (>=3.5.0)"] synapse = ["adlfs (>=2022.4.0)", "pyarrow (>=12.0.0)", "pyodbc (>=4.0.39)"] weaviate = ["weaviate-client (>=3.22)"] +[package.source] +type = "git" +url = "https://github.com/dlt-hub/dlt.git" +reference = "feat/1495-rename-JSONResponsePaginator" +resolved_reference = "762dd61a9ed18ebbbe2c16e2529173d02c6eb7ac" + [[package]] name = "dnspython" version = "2.4.2" @@ -1960,7 +1967,6 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -1969,7 +1975,6 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -1999,7 +2004,6 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -2008,7 +2012,6 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -4233,7 +4236,6 @@ files = [ {file = "pymongo-4.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6422b6763b016f2ef2beedded0e546d6aa6ba87910f9244d86e0ac7690f75c96"}, {file = "pymongo-4.5.0-cp312-cp312-win32.whl", hash = "sha256:77cfff95c1fafd09e940b3fdcb7b65f11442662fad611d0e69b4dd5d17a81c60"}, {file = "pymongo-4.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e57d859b972c75ee44ea2ef4758f12821243e99de814030f69a3decb2aa86807"}, - {file = "pymongo-4.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8443f3a8ab2d929efa761c6ebce39a6c1dca1c9ac186ebf11b62c8fe1aef53f4"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2b0176f9233a5927084c79ff80b51bd70bfd57e4f3d564f50f80238e797f0c8a"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89b3f2da57a27913d15d2a07d58482f33d0a5b28abd20b8e643ab4d625e36257"}, {file = "pymongo-4.5.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:5caee7bd08c3d36ec54617832b44985bd70c4cbd77c5b313de6f7fce0bb34f93"}, @@ -4568,7 +4570,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4576,16 +4577,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4602,7 +4595,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4610,7 +4602,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -6403,4 +6394,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "a81f4f33cbe1ad8fa08beeb89bdb28753e0e5f77c863c45aa1b98e6e6eff9614" +content-hash = "1a844410da7a56331db1b048f88ded4c25d3b6a21a0c82ab8b35e3e510a1b909" diff --git a/pyproject.toml b/pyproject.toml index 94043fd9b..cefe09666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = [{include = "sources"}] [tool.poetry.dependencies] python = ">=3.8.1,<3.13" -dlt = {version = "0.5.1", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]} +dlt = { git = "https://github.com/dlt-hub/dlt.git", branch = "feat/1495-rename-JSONResponsePaginator", extras = ["redshift", "bigquery", "postgres", "duckdb"]} graphlib-backport = {version = "*", python = "<3.9"} [tool.poetry.group.dev.dependencies] @@ -107,4 +107,4 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.black] -include = '.*py$' \ No newline at end of file +include = '.*py$' diff --git a/sources/rest_api/requirements.txt b/sources/rest_api/requirements.txt index 819f778f8..9f4ad4815 100644 --- a/sources/rest_api/requirements.txt +++ b/sources/rest_api/requirements.txt @@ -1 +1 @@ -dlt>=0.5.0 +dlt @ git+https://github.com/dlt-hub/dlt@feat/1495-rename-JSONResponsePaginator From 55a098eac61000c2ed3c6e4b1cac7604b1332ec1 Mon Sep 17 00:00:00 2001 From: Willi Date: Tue, 9 Jul 2024 14:09:17 +0530 Subject: [PATCH 4/6] dlt requirements check also accepts url with pointer to branch, tag, or commit --- check-requirements.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/check-requirements.py b/check-requirements.py index 675ee7699..d2d4b7fc3 100755 --- a/check-requirements.py +++ b/check-requirements.py @@ -23,6 +23,12 @@ "E.g. dlt>=0.3.5,<0.4.0" ) + +def has_url_with_pin(dlt_req: Requirement) -> bool: + """Checks if the url contains a reference to a branch, tag, or commit""" + return dlt_req.url is not None and "@" in dlt_req.url + + for source in source_dirs: req_path = source.joinpath("requirements.txt") if not req_path.is_file(): @@ -54,7 +60,7 @@ ) error = True continue - if not dlt_req.specifier: + if not dlt_req.specifier and not has_url_with_pin(dlt_req): print( f"ERROR: Source {source.name} dlt requirement '{dlt_req}' has no version constraint. {error_msg_suffix}" ) From 495e577483dd68facd8329d10a7c41736e7f8d1b Mon Sep 17 00:00:00 2001 From: Willi Date: Fri, 19 Jul 2024 12:40:03 +0530 Subject: [PATCH 5/6] bumps dlt dependency to have renamed JSONLinkPaginator --- poetry.lock | 21 +++++++-------------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8573aa7e4..8b6008687 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1063,26 +1063,25 @@ files = [ [[package]] name = "dlt" -version = "0.5.1" +version = "0.5.2a1" description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." optional = false python-versions = "<3.13,>=3.8.1" files = [ - {file = "dlt-0.5.1-py3-none-any.whl", hash = "sha256:19d7920816fadd049a1a92c4ecc4e740bf1cd5f6484fce535715f8c214a835ce"}, - {file = "dlt-0.5.1.tar.gz", hash = "sha256:dfa11e498feec3aca0022541850b6fa7dd3fddce4c2e6fef195530638269fec4"}, + {file = "dlt-0.5.2a1-py3-none-any.whl", hash = "sha256:755026dca845bf3a775060758ed61fc6907feb8b493774a82e2c8778bdd20ef7"}, + {file = "dlt-0.5.2a1.tar.gz", hash = "sha256:184912b3888915cfc12931d6a2d4b131d225f58b5eb07bc25eaf96a63872c8a9"}, ] -develop = false [package.dependencies] astunparse = ">=1.6.3" click = ">=7.1" duckdb = {version = ">=0.6.1,<0.11", optional = true, markers = "extra == \"duckdb\" or extra == \"motherduck\""} fsspec = ">=2022.4.0" -gcsfs = {version = ">=2022.4.0", optional = true} +gcsfs = {version = ">=2022.4.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\" or extra == \"gs\" or extra == \"clickhouse\""} gitpython = ">=3.1.29" giturlparse = ">=0.10.0" -google-cloud-bigquery = {version = ">=2.26.0", optional = true} -grpcio = {version = ">=1.50.0", optional = true} +google-cloud-bigquery = {version = ">=2.26.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\""} +grpcio = {version = ">=1.50.0", optional = true, markers = "extra == \"gcp\" or extra == \"bigquery\""} hexbytes = ">=0.2.2" humanize = ">=4.4.0" jsonpath-ng = ">=1.5.3" @@ -1132,12 +1131,6 @@ snowflake = ["snowflake-connector-python (>=3.5.0)"] synapse = ["adlfs (>=2022.4.0)", "pyarrow (>=12.0.0)", "pyodbc (>=4.0.39)"] weaviate = ["weaviate-client (>=3.22)"] -[package.source] -type = "git" -url = "https://github.com/dlt-hub/dlt.git" -reference = "feat/1495-rename-JSONResponsePaginator" -resolved_reference = "762dd61a9ed18ebbbe2c16e2529173d02c6eb7ac" - [[package]] name = "dnspython" version = "2.4.2" @@ -6440,4 +6433,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "1a844410da7a56331db1b048f88ded4c25d3b6a21a0c82ab8b35e3e510a1b909" +content-hash = "a554d522777de3fd04fe0cb73f3bedbd63edb4d72fa09a37dc6429ee60685458" diff --git a/pyproject.toml b/pyproject.toml index 7aadeee3a..ce0ed8f1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = [{include = "sources"}] [tool.poetry.dependencies] python = ">=3.8.1,<3.13" -dlt = { git = "https://github.com/dlt-hub/dlt.git", branch = "feat/1495-rename-JSONResponsePaginator", extras = ["redshift", "bigquery", "postgres", "duckdb"]} +dlt = {version = "0.5.2a1", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]} graphlib-backport = {version = "*", python = "<3.9"} [tool.poetry.group.dev.dependencies] From 9bbcbf30f6a4a1845828548ea80f6436d6a1ae84 Mon Sep 17 00:00:00 2001 From: Willi Date: Wed, 24 Jul 2024 14:13:32 +0530 Subject: [PATCH 6/6] bumps dlt version in rest_api/requirements.txt too --- sources/rest_api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/rest_api/requirements.txt b/sources/rest_api/requirements.txt index 9f4ad4815..b8f9c36ed 100644 --- a/sources/rest_api/requirements.txt +++ b/sources/rest_api/requirements.txt @@ -1 +1 @@ -dlt @ git+https://github.com/dlt-hub/dlt@feat/1495-rename-JSONResponsePaginator +dlt>=0.5.2a1