diff --git a/README.md b/README.md index ef115d6f..f6012f48 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,16 @@ Built with the [Meltano SDK](https://sdk.meltano.com) for Singer Taps and Target | Setting | Required | Default | Description | |:--------------------|:--------:|:-------:|:------------| -| sqlalchemy_url | True | None | SQLAlchemy connection string, example.`postgresql://postgres:postgres@localhost:5432/postgres` | -| default_target_schema | True | public | Postgres schema to send data to, example: tap-clickup | +| host | False | None | Hostname for postgres instance. Note if sqlalchemy_url is set this will be ignored. | +| port | False | 5432 | The port on which postgres is awaiting connection. Note if sqlalchemy_url is set this will be ignored. | +| user | False | None | User name used to authenticate. Note if sqlalchemy_url is set this will be ignored. | +| password | False | None | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. | +| database | False | None | Database name. Note if sqlalchemy_url is set this will be ignored. | +| sqlalchemy_url | False | None | SQLAlchemy connection string. This will override using host, user, password, port, dialect. Note that you must esacpe password special characters properly see https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords | +| dialect+driver | False | postgresql+psycopg2 | Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. | | stream_maps | False | None | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). | -| stream_map_config | False | None | User-defined config values to be used within map expressions. | | flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | +| stream_map_config | False | None | User-defined config values to be used within map expressions. | +| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | | flattening_max_depth| False | None | The max depth to flatten schemas. | A full list of supported settings and capabilities is available by running: `target-postgres --about` diff --git a/target_postgres/connector.py b/target_postgres/connector.py index 9e1c2965..b5a86352 100644 --- a/target_postgres/connector.py +++ b/target_postgres/connector.py @@ -1,10 +1,13 @@ """Connector class for target.""" from __future__ import annotations +from typing import cast + import sqlalchemy from singer_sdk import SQLConnector from singer_sdk import typing as th from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, JSONB +from sqlalchemy.engine import URL from sqlalchemy.types import TIMESTAMP @@ -30,6 +33,25 @@ def create_sqlalchemy_connection(self) -> sqlalchemy.engine.Connection: """ return self.create_sqlalchemy_engine().connect() + def get_sqlalchemy_url(self, config: dict) -> str: + """Generates a SQLAlchemy URL for sqlbuzz. + + Args: + config: The configuration for the connector. + """ + if config.get("sqlalchemy_url"): + return cast(str, config["sqlalchemy_url"]) + + else: + sqlalchemy_url = URL.create( + drivername=config["dialect+driver"], + username=config["user"], + password=config["password"], + host=config["host"], + database=config["database"], + ) + return cast(str, sqlalchemy_url) + def truncate_table(self, name): """Clear table data.""" self.connection.execute(f"TRUNCATE TABLE {name}") diff --git a/target_postgres/target.py b/target_postgres/target.py index 771d37ff..6eccb878 100644 --- a/target_postgres/target.py +++ b/target_postgres/target.py @@ -1,4 +1,7 @@ """Postgres target class.""" +from __future__ import annotations + +from pathlib import PurePath from singer_sdk import typing as th from singer_sdk.target_base import Target @@ -9,14 +12,104 @@ class TargetPostgres(Target): """Target for Postgres.""" + def __init__( + self, + config: dict | PurePath | str | list[PurePath | str] | None = None, + parse_env_config: bool = False, + validate_config: bool = True, + ) -> None: + """Initialize the target. + + Args: + config: Target configuration. Can be a dictionary, a single path to a + configuration file, or a list of paths to multiple configuration + files. + parse_env_config: Whether to look for configuration values in environment + variables. + validate_config: True to require validation of config settings. + """ + super().__init__( + config=config, + parse_env_config=parse_env_config, + validate_config=validate_config, + ) + # There's a few ways to do this in JSON Schema but it is schema draft dependent. + # https://stackoverflow.com/questions/38717933/jsonschema-attribute-conditionally-required + assert (self.config.get("sqlalchemy_url") is not None) or ( + self.config.get("host") is not None + and self.config.get("port") is not None + and self.config.get("user") is not None + and self.config.get("password") is not None + and self.config.get("nialect+driver") is not None + ), ( + "Need either the sqlalchemy_url to be set or host, port, user," + + "password, and dialect+driver to be set" + ) + name = "target-postgres" config_jsonschema = th.PropertiesList( + th.Property( + "host", + th.StringType, + description=( + "Hostname for postgres instance. " + + "Note if sqlalchemy_url is set this will be ignored." + ), + ), + th.Property( + "port", + th.IntegerType, + default=5432, + description=( + "The port on which postgres is awaiting connection. " + + "Note if sqlalchemy_url is set this will be ignored." + ), + ), + th.Property( + "user", + th.StringType, + description=( + "User name used to authenticate. " + + "Note if sqlalchemy_url is set this will be ignored.", + ), + ), + th.Property( + "password", + th.StringType, + description=( + "Password used to authenticate. " + "Note if sqlalchemy_url is set this will be ignored." + ), + ), + th.Property( + "database", + th.StringType, + description=( + "Database name. " + + "Note if sqlalchemy_url is set this will be ignored." + ), + ), th.Property( "sqlalchemy_url", th.StringType, - required=True, - description="SQLAlchemy connection string, example." - + "`postgresql://postgres:postgres@localhost:5432/postgres`", + description=( + "SQLAlchemy connection string. " + + "This will override using host, user, password, port," + + "dialect. Note that you must esacpe password special" + + "characters properly see" + + "https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords" # noqa: E501 + ), + ), + th.Property( + "dialect+driver", + th.StringType, + default="postgresql+psycopg2", + description=( + "Dialect+driver see " + + "https://docs.sqlalchemy.org/en/20/core/engines.html. " + + "Generally just leave this alone. " + + "Note if sqlalchemy_url is set this will be ignored." + ), ), th.Property( "default_target_schema", diff --git a/target_postgres/tests/test_standard_target.py b/target_postgres/tests/test_standard_target.py index e29e781b..bc15d972 100644 --- a/target_postgres/tests/test_standard_target.py +++ b/target_postgres/tests/test_standard_target.py @@ -18,8 +18,11 @@ @pytest.fixture(scope="session") def postgres_config(): return { - "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres", - "default_target_schema": f"pytest_{str(uuid.uuid4()).replace('-','_')}", + "dialect+driver": "postgresql+psycopg2", + "host": "localhost", + "user": "postgres", + "password": "postgres", + "database": "postgres", } @@ -51,6 +54,18 @@ def singer_file_to_target(file_name, target) -> None: # TODO should set schemas for each tap individually so we don't collide + + +def test_sqlalchemy_url_config(): + """Be sure that passing a sqlalchemy_url works""" + config = { + "sqlalchemy_url": "postgresql://postgres:postgres@localhost:5432/postgres" + } + tap = SampleTapCountries(config={}, state=None) + target = TargetPostgres(config=config) + sync_end_to_end(tap, target) + + # Test name would work well def test_countries_to_postgres(postgres_config): tap = SampleTapCountries(config={}, state=None)