Merge branch 'main' into http-connector

Author: edgarrmondragon

.github/workflows/constraints.txt

@@ -1,4 +1,4 @@
-pip==23.2
+pip==23.2.1
 poetry==1.5.1
 pre-commit==3.3.3
 nox==2023.4.22

.github/workflows/test.yml

@@ -37,7 +37,7 @@ env:
 
 jobs:
   tests:
-    name: Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }}
+    name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}"
     runs-on: ${{ matrix.os }}
     env:
       NOXSESSION: ${{ matrix.session }}
@@ -47,9 +47,11 @@ jobs:
         session: [tests]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        sqlalchemy: ["2.*"]
         include:
-        - { session: doctest, python-version: "3.10", os: "ubuntu-latest" }
-        - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest" }
+        - { session: tests,   python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1.*" }
+        - { session: doctest, python-version: "3.10", os: "ubuntu-latest", sqlalchemy: "2.*" }
+        - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest", sqlalchemy: "2.*" }
 
     steps:
     - name: Check out the repository
@@ -86,6 +88,8 @@ jobs:
         nox --version
 
     - name: Run Nox
+      env:
+        SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }}
       run: |
         nox --python=${{ matrix.python-version }}
 

.pre-commit-config.yaml

@@ -36,14 +36,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.23.2
+  rev: 0.23.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.277
+  rev: v0.0.280
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

docs/deprecation.md

@@ -11,3 +11,5 @@ incompatible way, following their deprecation, as indicated in the
   [`RESTStream.get_new_paginator`](singer_sdk.RESTStream.get_new_paginator).
 
   See the [migration guide](./guides/pagination-classes.md) for more information.
+
+- The `singer_sdk.testing.get_standard_tap_tests` and `singer_sdk.testing.get_standard_target_tests` functions will be removed. Replace them with `singer_sdk.testing.get_tap_test_class` and `singer_sdk.testing.get_target_test_class` functions respective to generate a richer test suite.

noxfile.py

@@ -86,6 +86,14 @@ def tests(session: Session) -> None:
     session.install(".[s3]")
     session.install(*test_dependencies)
 
+    sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
+    if sqlalchemy_version:
+        # Bypass nox-poetry use of --constraint so we can install a version of
+        # SQLAlchemy that doesn't match what's in poetry.lock.
+        session.poetry.session.install(  # type: ignore[attr-defined]
+            f"sqlalchemy=={sqlalchemy_version}",
+        )
+
     try:
         session.run(
             "coverage",
@@ -96,9 +104,6 @@ def tests(session: Session) -> None:
             "-v",
             "--durations=10",
             *session.posargs,
-            env={
-                "SQLALCHEMY_WARN_20": "1",
-            },
         )
     finally:
         if session.interactive:

poetry.lock

@@ -52,7 +52,7 @@ memoization = ">=0.3.2,<0.5.0"
 jsonpath-ng = "^1.5.3"
 joblib = "^1.0.1"
 inflection = "^0.5.1"
-sqlalchemy = "^1.4"
+sqlalchemy = ">=1.4,<3.0"
 python-dotenv = ">=0.20,<0.22"
 typing-extensions = "^4.2.0"
 simplejson = "^3.17.6"
@@ -109,7 +109,6 @@ numpy = [
     { version = ">=1.22", python = ">=3.8" },
 ]
 requests-mock = "^1.10.0"
-sqlalchemy2-stubs = {version = "^0.0.2a32", allow-prereleases = true}
 types-jsonschema = "^4.17.0.6"
 types-python-dateutil = "^2.8.19"
 types-pytz = ">=2022.7.1.2,<2024.0.0.0"
@@ -133,9 +132,6 @@ exclude = ".*simpleeval.*"
 
 [tool.pytest.ini_options]
 addopts = '-vvv --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"'
-filterwarnings = [
-    "error::sqlalchemy.exc.RemovedIn20Warning",
-]
 markers = [
     "external: Tests relying on external resources",
     "windows: Tests that only run on Windows",
@@ -191,9 +187,6 @@ fail_under = 82
 [tool.mypy]
 exclude = "tests"
 files = "singer_sdk"
-plugins = [
-    "sqlalchemy.ext.mypy.plugin",
-]
 python_version = "3.8"
 warn_unused_configs = true
 warn_unused_ignores = true
@@ -216,11 +209,13 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
-pytest11 = { callable = "singer_sdk:testing.pytest_plugin", extras = ["testing"] }
+pytest11 = { reference = "singer_sdk:testing.pytest_plugin", extras = ["testing"], type = "console" }
 
 [tool.ruff]
 exclude = [
     "cookiecutter/*",
+    "singer_sdk/helpers/_simpleeval.py",
+    "tests/core/test_simpleeval.py",
 ]
 ignore = [
     "ANN101",  # Missing type annotation for `self` in method

pyproject.toml

@@ -197,7 +197,7 @@ def get_sqlalchemy_url(self, config: t.Mapping[str, t.Any]) -> str:
     @staticmethod
     def to_jsonschema_type(
         sql_type: (
-            str
+            str  # noqa: ANN401
             | sqlalchemy.types.TypeEngine
             | type[sqlalchemy.types.TypeEngine]
             | t.Any

singer_sdk/connectors/sql.py

@@ -11,4 +11,9 @@
     from importlib import metadata
     from typing import Protocol, final  # noqa: ICN003
 
-__all__ = ["metadata", "final", "Protocol"]
+if sys.version_info < (3, 9):
+    import importlib_resources as resources
+else:
+    from importlib import resources
+
+__all__ = ["metadata", "final", "resources", "Protocol"]

singer_sdk/helpers/_compat.py

@@ -252,15 +252,15 @@ def _flatten_schema(  # noqa: C901
             else:
                 items.append((new_key, v))
         elif len(v.values()) > 0:
-            if list(v.values())[0][0]["type"] == "string":
-                list(v.values())[0][0]["type"] = ["null", "string"]
-                items.append((new_key, list(v.values())[0][0]))
-            elif list(v.values())[0][0]["type"] == "array":
-                list(v.values())[0][0]["type"] = ["null", "array"]
-                items.append((new_key, list(v.values())[0][0]))
-            elif list(v.values())[0][0]["type"] == "object":
-                list(v.values())[0][0]["type"] = ["null", "object"]
-                items.append((new_key, list(v.values())[0][0]))
+            if next(iter(v.values()))[0]["type"] == "string":
+                next(iter(v.values()))[0]["type"] = ["null", "string"]
+                items.append((new_key, next(iter(v.values()))[0]))
+            elif next(iter(v.values()))[0]["type"] == "array":
+                next(iter(v.values()))[0]["type"] = ["null", "array"]
+                items.append((new_key, next(iter(v.values()))[0]))
+            elif next(iter(v.values()))[0]["type"] == "object":
+                next(iter(v.values()))[0]["type"] = ["null", "object"]
+                items.append((new_key, next(iter(v.values()))[0]))
 
     # Sort and check for duplicates
     def _key_func(item):

singer_sdk/helpers/_flattening.py

@@ -215,6 +215,9 @@ def datetime_error_treatment(self) -> DatetimeErrorTreatmentEnum:
     def key_properties(self) -> list[str]:
         """Return key properties.
 
+        Override this method to return a list of key properties in a format that is
+        compatible with the target.
+
         Returns:
             A list of stream key properties.
         """
@@ -331,10 +334,10 @@ def _singer_validate_message(self, record: dict) -> None:
         Raises:
             MissingKeyPropertiesError: If record is missing one or more key properties.
         """
-        if not all(key_property in record for key_property in self.key_properties):
+        if any(key_property not in record for key_property in self._key_properties):
             msg = (
                 f"Record is missing one or more key_properties. \n"
-                f"Key Properties: {self.key_properties}, "
+                f"Key Properties: {self._key_properties}, "
                 f"Record Keys: {list(record.keys())}"
             )
             raise MissingKeyPropertiesError(

singer_sdk/sinks/core.py

@@ -322,15 +322,20 @@ def bulk_insert_records(
         if isinstance(insert_sql, str):
             insert_sql = sqlalchemy.text(insert_sql)
 
-        conformed_records = (
-            [self.conform_record(record) for record in records]
-            if isinstance(records, list)
-            else (self.conform_record(record) for record in records)
-        )
+        conformed_records = [self.conform_record(record) for record in records]
+        property_names = list(self.conform_schema(schema)["properties"].keys())
+
+        # Create new record dicts with missing properties filled in with None
+        new_records = [
+            {name: record.get(name) for name in property_names}
+            for record in conformed_records
+        ]
+
         self.logger.info("Inserting with SQL: %s", insert_sql)
         with self.connector.connect() as conn, conn.begin():
-            conn.execute(insert_sql, conformed_records)
-        return len(conformed_records) if isinstance(conformed_records, list) else None
+            result = conn.execute(insert_sql, new_records)
+
+        return result.rowcount
 
     def merge_upsert_from_table(
         self,

singer_sdk/sinks/sql.py

@@ -217,7 +217,10 @@ def is_timestamp_replication_key(self) -> bool:
         type_dict = self.schema.get("properties", {}).get(self.replication_key)
         return is_datetime_type(type_dict)
 
-    def get_starting_replication_key_value(self, context: dict | None) -> t.Any | None:
+    def get_starting_replication_key_value(
+        self,
+        context: dict | None,
+    ) -> t.Any | None:  # noqa: ANN401
         """Get starting replication key.
 
         Will return the value of the stream's replication key when `--state` is passed.
@@ -385,7 +388,7 @@ def _write_starting_replication_value(self, context: dict | None) -> None:
     def get_replication_key_signpost(
         self,
         context: dict | None,  # noqa: ARG002
-    ) -> datetime.datetime | t.Any | None:
+    ) -> datetime.datetime | t.Any | None:  # noqa: ANN401
         """Get the replication signpost.
 
         For timestamp-based replication keys, this defaults to `utc_now()`. For
@@ -1255,7 +1258,7 @@ def get_child_context(self, record: dict, context: dict | None) -> dict | None:
 
         Raises:
             NotImplementedError: If the stream has children but this method is not
-                overriden.
+                overridden.
         """
         if context is None:
             for child_stream in self.child_streams:

singer_sdk/streams/core.py

@@ -8,8 +8,10 @@
 from singer_sdk.helpers._classproperty import classproperty
 from singer_sdk.streams.rest import RESTStream
 
+_TToken = t.TypeVar("_TToken")
 
-class GraphQLStream(RESTStream, metaclass=abc.ABCMeta):
+
+class GraphQLStream(RESTStream, t.Generic[_TToken], metaclass=abc.ABCMeta):
     """Abstract base class for API-type streams.
 
     GraphQL streams inherit from the class `GraphQLStream`, which in turn inherits from
@@ -43,7 +45,7 @@ def query(self) -> str:
     def prepare_request_payload(
         self,
         context: dict | None,
-        next_page_token: t.Any | None,
+        next_page_token: _TToken | None,
     ) -> dict | None:
         """Prepare the data payload for the GraphQL API request.
 

singer_sdk/streams/graphql.py

@@ -612,37 +612,17 @@ class SQLTap(Tap):
     # Stream class used to initialize new SQL streams from their catalog declarations.
     default_stream_class: type[SQLStream]
 
-    def __init__(
-        self,
-        *,
-        config: dict | PurePath | str | list[PurePath | str] | None = None,
-        catalog: PurePath | str | dict | None = None,
-        state: PurePath | str | dict | None = None,
-        parse_env_config: bool = False,
-        validate_config: bool = True,
-    ) -> None:
+    def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
         """Initialize the SQL tap.
 
         The SQLTap initializer additionally creates a cache variable for _catalog_dict.
 
         Args:
-            config: Tap configuration. Can be a dictionary, a single path to a
-                configuration file, or a list of paths to multiple configuration
-                files.
-            catalog: Tap catalog. Can be a dictionary or a path to the catalog file.
-            state: Tap state. Can be dictionary or a path to the state file.
-            parse_env_config: Whether to look for configuration values in environment
-                variables.
-            validate_config: True to require validation of config settings.
+            *args: Positional arguments for the Tap initializer.
+            **kwargs: Keyword arguments for the Tap initializer.
         """
         self._catalog_dict: dict | None = None
-        super().__init__(
-            config=config,
-            catalog=catalog,
-            state=state,
-            parse_env_config=parse_env_config,
-            validate_config=validate_config,
-        )
+        super().__init__(*args, **kwargs)
 
     @property
     def catalog_dict(self) -> dict:

singer_sdk/tap_base.py

@@ -2,27 +2,57 @@
 
 from __future__ import annotations
 
+import typing as t
+import warnings
+
 from .config import SuiteConfig
 from .factory import get_tap_test_class, get_target_test_class
 from .legacy import (
     _get_tap_catalog,
     _select_all,
-    get_standard_tap_tests,
-    get_standard_target_tests,
     sync_end_to_end,
     tap_sync_test,
     tap_to_target_sync_test,
     target_sync_test,
 )
 from .runners import SingerTestRunner, TapTestRunner, TargetTestRunner
 
+
+def __getattr__(name: str) -> t.Any:  # noqa: ANN401
+    if name == "get_standard_tap_tests":
+        warnings.warn(
+            "The function singer_sdk.testing.get_standard_tap_tests is deprecated "
+            "and will be removed in a future release. Use get_tap_test_class instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        from .legacy import get_standard_tap_tests
+
+        return get_standard_tap_tests
+
+    if name == "get_standard_target_tests":
+        warnings.warn(
+            "The function singer_sdk.testing.get_standard_target_tests is deprecated "
+            "and will be removed in a future release. Use get_target_test_class "
+            "instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        from .legacy import get_standard_target_tests
+
+        return get_standard_target_tests
+
+    msg = f"module {__name__} has no attribute {name}"
+    raise AttributeError(msg)
+
+
 __all__ = [
     "get_tap_test_class",
     "get_target_test_class",
     "_get_tap_catalog",
     "_select_all",
-    "get_standard_tap_tests",
-    "get_standard_target_tests",
     "sync_end_to_end",
     "tap_sync_test",
     "tap_to_target_sync_test",