feat: Create SQLAlchemy URL from config take 2

README.md

@@ -14,10 +14,16 @@ Built with the [Meltano SDK](https://sdk.meltano.com) for Singer Taps and Target
 
 | Setting             | Required | Default | Description |
 |:--------------------|:--------:|:-------:|:------------|
-| sqlalchemy_url      | True     | None    | SQLAlchemy connection string, example.`postgresql://postgres:postgres@localhost:5432/postgres` |
-| default_target_schema       | True     | public  | Postgres schema to send data to, example: tap-clickup |
+| host                | False    | None    | Hostname for postgres instance. Note if sqlalchemy_url is set this will be ignored. |
+| port                | False    | 5432 | The port on which postgres is awaiting connection. Note if sqlalchemy_url is set this will be ignored. |
+| user                | False    | None    | User name used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
+| password            | False    | None    | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. |
+| database            | False    | None    | Database name. Note if sqlalchemy_url is set this will be ignored. |
+| sqlalchemy_url      | False    | None    | SQLAlchemy connection string. This will override using host, user, password, port, dialect. Note that you must esacpe password special characters properly see https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords |
+| dialect+driver    | False    | postgresql+psycopg2 | Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. |
 | stream_maps         | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
-| stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |                                                                                                                                 | flattening_enabled  | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
+| stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |
+| flattening_enabled  | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
 | flattening_max_depth| False    | None    | The max depth to flatten schemas. |
 
 A full list of supported settings and capabilities is available by running: `target-postgres --about`

target_postgres/connector.py

@@ -1,10 +1,13 @@
 """Connector class for target."""
 from __future__ import annotations
 
+from typing import cast
+
 import sqlalchemy
 from singer_sdk import SQLConnector
 from singer_sdk import typing as th
 from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, JSONB
+from sqlalchemy.engine import URL
 from sqlalchemy.types import TIMESTAMP
 
 
@@ -30,6 +33,25 @@ def create_sqlalchemy_connection(self) -> sqlalchemy.engine.Connection:
         """
         return self.create_sqlalchemy_engine().connect()
 
+    def get_sqlalchemy_url(self, config: dict) -> str:
+        """Generates a SQLAlchemy URL for sqlbuzz.
+
+        Args:
+            config: The configuration for the connector.
+        """
+        if config.get("sqlalchemy_url"):
+            return cast(str, config["sqlalchemy_url"])
+
+        else:
+            sqlalchemy_url = URL.create(
+                drivername=config["dialect+driver"],
+                username=config["user"],
+                password=config["password"],
+                host=config["host"],
+                database=config["database"],
+            )
+            return cast(str, sqlalchemy_url)
+
     def truncate_table(self, name):
         """Clear table data."""
         self.connection.execute(f"TRUNCATE TABLE {name}")

target_postgres/target.py

@@ -1,4 +1,7 @@
 """Postgres target class."""
+from __future__ import annotations
+
+from pathlib import PurePath
 
 from singer_sdk import typing as th
 from singer_sdk.target_base import Target
@@ -9,14 +12,104 @@
 class TargetPostgres(Target):
     """Target for Postgres."""
 
+    def __init__(
+        self,
+        config: dict | PurePath | str | list[PurePath | str] | None = None,
+        parse_env_config: bool = False,
+        validate_config: bool = True,
+    ) -> None:
+        """Initialize the target.
+
+        Args:
+            config: Target configuration. Can be a dictionary, a single path to a
+                configuration file, or a list of paths to multiple configuration
+                files.
+            parse_env_config: Whether to look for configuration values in environment
+                variables.
+            validate_config: True to require validation of config settings.
+        """
+        super().__init__(
+            config=config,
+            parse_env_config=parse_env_config,
+            validate_config=validate_config,
+        )
+        # There's a few ways to do this in JSON Schema but it is schema draft dependent.
+        # https://stackoverflow.com/questions/38717933/jsonschema-attribute-conditionally-required
+        assert (self.config.get("sqlalchemy_url") is not None) or (
+            self.config.get("host") is not None
+            and self.config.get("port") is not None
+            and self.config.get("user") is not None
+            and self.config.get("password") is not None
+            and self.config.get("nialect+driver") is not None
+        ), (
+            "Need either the sqlalchemy_url to be set or host, port, user,"
+            + "password, and dialect+driver to be set"
+        )
+
     name = "target-postgres"
     config_jsonschema = th.PropertiesList(
+        th.Property(
+            "host",
+            th.StringType,
+            description=(
+                "Hostname for postgres instance. "
+                + "Note if sqlalchemy_url is set this will be ignored."
+            ),
+        ),
+        th.Property(
+            "port",
+            th.IntegerType,
+            default=5432,
+            description=(
+                "The port on which postgres is awaiting connection. "
+                + "Note if sqlalchemy_url is set this will be ignored."
+            ),
+        ),
+        th.Property(
+            "user",
+            th.StringType,
+            description=(
+                "User name used to authenticate. "
+                + "Note if sqlalchemy_url is set this will be ignored.",
+            ),
+        ),
+        th.Property(
+            "password",
+            th.StringType,
+            description=(
+                "Password used to authenticate. "
+                "Note if sqlalchemy_url is set this will be ignored."
+            ),
+        ),
+        th.Property(
+            "database",
+            th.StringType,
+            description=(
+                "Database name. "
+                + "Note if sqlalchemy_url is set this will be ignored."
+            ),
+        ),
         th.Property(
             "sqlalchemy_url",
             th.StringType,
-            required=True,
-            description="SQLAlchemy connection string, example."
-            + "`postgresql://postgres:postgres@localhost:5432/postgres`",
+            description=(
+                "SQLAlchemy connection string. "
+                + "This will override using host, user, password, port,"
+                + "dialect. Note that you must esacpe password special"
+                + "characters properly see"
+                + "https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords"  # noqa: E501
+            ),
+        ),
+        th.Property(
+            "dialect+driver",
+            th.StringType,
+            default="postgresql+psycopg2",
+            description=(
+                "Dialect+driver see "
+                + "https://docs.sqlalchemy.org/en/20/core/engines.html. "
+                + "Generally just leave this alone. "
+                + "Note if sqlalchemy_url is set this will be ignored."
+            ),
         ),
         th.Property(
             "default_target_schema",

target_postgres/tests/test_standard_target.py

@@ -18,8 +18,11 @@
 @pytest.fixture(scope="session")
 def postgres_config():
     return {
-        "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres",
-        "default_target_schema": f"pytest_{str(uuid.uuid4()).replace('-','_')}",
+        "dialect+driver": "postgresql+psycopg2",
+        "host": "localhost",
+        "user": "postgres",
+        "password": "postgres",
+        "database": "postgres",
     }
 
 
@@ -51,6 +54,18 @@ def singer_file_to_target(file_name, target) -> None:
 
 
 # TODO should set schemas for each tap individually so we don't collide
+
+
+def test_sqlalchemy_url_config():
+    """Be sure that passing a sqlalchemy_url works"""
+    config = {
+        "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres"
+    }
+    tap = SampleTapCountries(config={}, state=None)
+    target = TargetPostgres(config=config)
+    sync_end_to_end(tap, target)
+
+
 # Test name would work well
 def test_countries_to_postgres(postgres_config):
     tap = SampleTapCountries(config={}, state=None)