rest_api: Renames JSONResponsePaginator to JSONLinkPaginator. (#523)

* rest_api: Renames JSONResponsePaginator to JSONLinkPaginator.

* bumps dlt dependency to have renamed JSONLinkPaginator

* dlt requirements check also accepts url with pointer to branch, tag, or commit so that CI checks pass when we depend on an open PR in dlt-core

Author: willi-mueller

check-requirements.py

@@ -23,6 +23,12 @@
     "E.g. dlt>=0.3.5,<0.4.0"
 )
 
+
+def has_url_with_pin(dlt_req: Requirement) -> bool:
+    """Checks if the url contains a reference to a branch, tag, or commit"""
+    return dlt_req.url is not None and "@" in dlt_req.url
+
+
 for source in source_dirs:
     req_path = source.joinpath("requirements.txt")
     if not req_path.is_file():
@@ -54,7 +60,7 @@
         )
         error = True
         continue
-    if not dlt_req.specifier:
+    if not dlt_req.specifier and not has_url_with_pin(dlt_req):
         print(
             f"ERROR: Source {source.name} dlt requirement '{dlt_req}' has no version constraint. {error_msg_suffix}"
         )

poetry.lock

@@ -12,7 +12,7 @@ packages = [{include = "sources"}]
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.13"
-dlt = {version = "0.5.1", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
+dlt = {version = "0.5.2a1", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb"]}
 graphlib-backport = {version = "*", python = "<3.9"}
 
 [tool.poetry.group.dev.dependencies]
@@ -108,4 +108,4 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.black]
-include = '.*py$'
+include = '.*py$'

pyproject.toml

@@ -130,14 +130,14 @@ Possible paginators are:
 | Paginator | String Alias | Note |
 | --------- | ------------ | ---- |
 | BasePaginator | | |
-| HeaderLinkPaginator | `header_links` | |
-| JSONResponsePaginator | `json_links` | The pagination metainformation is in a node of the JSON response (see example below) |
+| HeaderLinkPaginator | `header_link` | |
+| JSONLinkPaginator | `json_link` | The pagination metainformation is in a node of the JSON response (see example below) |
 | SinglePagePaginator | `single_page` | The response will be interpreted as a single-page response, ignoring possible pagination metadata |
 
-Usage example of the `JSONResponsePaginator`, for a response with the URL of the next page located at `paging.next`:
+Usage example of the `JSONLinkPaginator`, for a response with the URL of the next page located at `paging.next`:
 ```python
-"paginator": JSONResponsePaginator(
-    next_key=["paging", "next"]
+"paginator": JSONLinkPaginator(
+    next_url_path="paging.next"]
 )
 ```
 
@@ -215,4 +215,4 @@ Resources with the name different from the endpoint string will be:
 ```
 In case you need to have a resource with a name different from the table created, you can pass the property `table_name` too.
 
-For the other properties, see the [resource_defaults](#resource_defaults) above.
+For the other properties, see the [resource_defaults](#resource_defaults) above.

sources/rest_api/README.md

@@ -79,7 +79,7 @@ def rest_api_source(
         pokemon_source = rest_api_source({
             "client": {
                 "base_url": "https://pokeapi.co/api/v2/",
-                "paginator": "json_response",
+                "paginator": "json_link",
             },
             "endpoints": {
                 "pokemon": {

sources/rest_api/__init__.py

@@ -29,7 +29,7 @@
     BasePaginator,
     SinglePagePaginator,
     HeaderLinkPaginator,
-    JSONResponsePaginator,
+    JSONLinkPaginator,
     JSONResponseCursorPaginator,
     OffsetPaginator,
     PageNumberPaginator,
@@ -60,7 +60,7 @@
 
 
 PAGINATOR_MAP: Dict[PaginatorType, Type[BasePaginator]] = {
-    "json_response": JSONResponsePaginator,
+    "json_link": JSONLinkPaginator,
     "header_link": HeaderLinkPaginator,
     "auto": None,
     "single_page": SinglePagePaginator,

sources/rest_api/config_setup.py

@@ -1 +1 @@
-dlt>=0.4.11
+dlt>=0.5.2a1

sources/rest_api/requirements.txt

@@ -30,7 +30,7 @@
 from dlt.sources.helpers.rest_client.paginators import (
     SinglePagePaginator,
     HeaderLinkPaginator,
-    JSONResponsePaginator,
+    JSONLinkPaginator,
     JSONResponseCursorPaginator,
     OffsetPaginator,
     PageNumberPaginator,
@@ -43,7 +43,7 @@
 )
 
 PaginatorType = Literal[
-    "json_response",
+    "json_link",
     "header_link",
     "auto",
     "single_page",
@@ -84,7 +84,7 @@ class HeaderLinkPaginatorConfig(PaginatorTypeConfig, total=False):
     links_next_key: Optional[str]
 
 
-class JSONResponsePaginatorConfig(PaginatorTypeConfig, total=False):
+class JSONLinkPaginatorConfig(PaginatorTypeConfig, total=False):
     """Locates the next page URL within the JSON response body. The key
     containing the URL can be specified using a JSON path."""
 
@@ -104,12 +104,12 @@ class JSONResponseCursorPaginatorConfig(PaginatorTypeConfig, total=False):
     PageNumberPaginatorConfig,
     OffsetPaginatorConfig,
     HeaderLinkPaginatorConfig,
-    JSONResponsePaginatorConfig,
+    JSONLinkPaginatorConfig,
     JSONResponseCursorPaginatorConfig,
     BasePaginator,
     SinglePagePaginator,
     HeaderLinkPaginator,
-    JSONResponsePaginator,
+    JSONLinkPaginator,
     JSONResponseCursorPaginator,
     OffsetPaginator,
     PageNumberPaginator,

sources/rest_api/typing.py

@@ -116,7 +116,7 @@ def load_pokemon() -> None:
             "client": {
                 "base_url": "https://pokeapi.co/api/v2/",
                 # If you leave out the paginator, it will be inferred from the API:
-                # paginator: "json_response",
+                # paginator: "json_link",
             },
             "resource_defaults": {
                 "endpoint": {

sources/rest_api_pipeline.py

@@ -137,7 +137,7 @@ class CustomOAuthAuth(OAuth2AuthBase):
                     "params": {
                         "limit": 100,
                     },
-                    "paginator": "json_response",
+                    "paginator": "json_link",
                 },
             },
         ],
@@ -217,7 +217,7 @@ class CustomOAuthAuth(OAuth2AuthBase):
                             "initial_value": "2024-01-25T11:21:28Z",
                         },
                     },
-                    "paginator": "json_response",
+                    "paginator": "json_link",
                 },
             },
         ],
@@ -232,7 +232,7 @@ class CustomOAuthAuth(OAuth2AuthBase):
                     "params": {
                         "limit": 100,
                     },
-                    "paginator": "json_response",
+                    "paginator": "json_link",
                     "incremental": {
                         "start_param": "since",
                         "end_param": "until",
@@ -313,7 +313,7 @@ class CustomOAuthAuth(OAuth2AuthBase):
     {"type": "page_number", "page": 10, "base_page": 1, "total_path": "response.pages"},
     {"type": "offset", "limit": 100, "maximum_offset": 1000},
     {"type": "header_link", "links_next_key": "next_page"},
-    {"type": "json_response", "next_url_path": "response.nex_page_link"},
+    {"type": "json_link", "next_url_path": "response.nex_page_link"},
     {"type": "cursor", "cursor_param": "cursor"},
 ]