Skip to content

Commit 16e05f3

Browse files
taliawsmileytaliawsmiley
authored
feat: Add SDK pytests (#227)
With this PR, almost all SDK tests are now included in the target-postgres testing suite. Exceptions that are excluded are noted as comments. Also, pytest files are restructured to allow for further tests (and test files) to be added more easily. Closes #223
1 parent 8262aeb commit 16e05f3

File tree

3 files changed

+155
-53
lines changed

3 files changed

+155
-53
lines changed

‎target_postgres/tests/core.py

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
""" Config and base values for target-postgres testing """
2+
# flake8: noqa
3+
import sqlalchemy
4+
5+
from target_postgres.target import TargetPostgres
6+
7+
8+
def postgres_config():
9+
return {
10+
"dialect+driver": "postgresql+psycopg2",
11+
"host": "localhost",
12+
"user": "postgres",
13+
"password": "postgres",
14+
"database": "postgres",
15+
"port": 5432,
16+
"ssl_enable": True,
17+
"ssl_client_certificate_enable": True,
18+
"ssl_mode": "verify-full",
19+
"ssl_certificate_authority": "./ssl/root.crt",
20+
"ssl_client_certificate": "./ssl/cert.crt",
21+
"ssl_client_private_key": "./ssl/pkey.key",
22+
"ssl_storage_directory": ".secrets",
23+
"add_record_metadata": True,
24+
"hard_delete": False,
25+
"default_target_schema": "melty",
26+
}
27+
28+
29+
def postgres_config_no_ssl():
30+
return {
31+
"dialect+driver": "postgresql+psycopg2",
32+
"host": "localhost",
33+
"user": "postgres",
34+
"password": "postgres",
35+
"database": "postgres",
36+
"port": 5433,
37+
"add_record_metadata": True,
38+
"hard_delete": False,
39+
"default_target_schema": "melty",
40+
}
41+
42+
43+
def postgres_config_ssh_tunnel():
44+
return {
45+
"sqlalchemy_url": "postgresql://postgres:[email protected]:5432/main",
46+
"ssh_tunnel": {
47+
"enable": True,
48+
"host": "127.0.0.1",
49+
"port": 2223,
50+
"username": "melty",
51+
"private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----", # noqa: E501
52+
},
53+
}
54+
55+
56+
def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine:
57+
return TargetPostgres.default_sink_class.connector_class(
58+
config=target_postgres.config
59+
)._engine

‎target_postgres/tests/test_sdk.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
""" SDK tests for target postgres """
2+
# flake8: noqa
3+
import pytest
4+
from singer_sdk.testing import get_target_test_class
5+
from singer_sdk.testing.suites import TestSuite
6+
from singer_sdk.testing.target_tests import (
7+
TargetArrayData,
8+
TargetCamelcaseComplexSchema,
9+
TargetCamelcaseTest,
10+
TargetCliPrintsTest,
11+
TargetDuplicateRecords,
12+
TargetEncodedStringData,
13+
TargetInvalidSchemaTest,
14+
TargetMultipleStateMessages,
15+
TargetNoPrimaryKeys,
16+
TargetOptionalAttributes,
17+
TargetRecordBeforeSchemaTest,
18+
TargetRecordMissingKeyProperty,
19+
TargetRecordMissingOptionalFields,
20+
TargetRecordMissingRequiredProperty,
21+
TargetSchemaNoProperties,
22+
TargetSchemaUpdates,
23+
TargetSpecialCharsInAttributes,
24+
)
25+
26+
from target_postgres.target import TargetPostgres
27+
28+
from .core import create_engine, postgres_config
29+
30+
target_tests = TestSuite(
31+
kind="target",
32+
tests=[
33+
TargetArrayData,
34+
TargetCamelcaseComplexSchema,
35+
TargetCamelcaseTest,
36+
TargetCliPrintsTest,
37+
TargetDuplicateRecords,
38+
# Postgres doesn't support NULL characters in strings
39+
# TargetEncodedStringData,
40+
TargetInvalidSchemaTest,
41+
# This tap only outputs one state message at the end of execution, fails assertion.
42+
# Separate custom test in test_target_postgres.py
43+
# TargetMultipleStateMessages,
44+
TargetNoPrimaryKeys,
45+
TargetOptionalAttributes,
46+
TargetRecordBeforeSchemaTest,
47+
TargetRecordMissingKeyProperty,
48+
# Fails, but gives appropriate error message
49+
# TargetRecordMissingRequiredProperty,
50+
TargetSchemaNoProperties,
51+
TargetSchemaUpdates,
52+
TargetSpecialCharsInAttributes,
53+
TargetRecordMissingOptionalFields,
54+
],
55+
)
56+
57+
58+
class BasePostgresSDKTests:
59+
"""Base class for Postgres SDK tests."""
60+
61+
@pytest.fixture()
62+
def connection(self, runner):
63+
engine = create_engine(runner)
64+
return engine.connect()
65+
66+
67+
SDKTests = get_target_test_class(
68+
target_class=TargetPostgres,
69+
config=postgres_config(),
70+
custom_suites=[target_tests],
71+
suite_config=None,
72+
include_target_tests=False,
73+
)
74+
75+
76+
class TestTargetPostgres(BasePostgresSDKTests, SDKTests):
77+
"""SDK tests"""

‎target_postgres/tests/test_standard_target.py renamed to ‎target_postgres/tests/test_target_postgres.py

Lines changed: 19 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
""" Attempt at making some standard Target Tests. """
1+
""" Postgres target tests """
22
# flake8: noqa
33
import copy
44
import io
5-
import uuid
65
from contextlib import redirect_stdout
76
from decimal import Decimal
87
from pathlib import Path
@@ -11,7 +10,7 @@
1110
import pytest
1211
import sqlalchemy
1312
from singer_sdk.exceptions import MissingKeyPropertiesError
14-
from singer_sdk.testing import sync_end_to_end
13+
from singer_sdk.testing import get_target_test_class, sync_end_to_end
1514
from sqlalchemy.dialects.postgresql import ARRAY
1615
from sqlalchemy.types import TEXT, TIMESTAMP
1716

@@ -22,68 +21,35 @@
2221
SampleTapCountries,
2322
)
2423

24+
from .core import (
25+
create_engine,
26+
postgres_config,
27+
postgres_config_no_ssl,
28+
postgres_config_ssh_tunnel,
29+
)
2530

26-
@pytest.fixture(scope="session")
27-
def postgres_config():
28-
return {
29-
"dialect+driver": "postgresql+psycopg2",
30-
"host": "localhost",
31-
"user": "postgres",
32-
"password": "postgres",
33-
"database": "postgres",
34-
"port": 5432,
35-
"ssl_enable": True,
36-
"ssl_client_certificate_enable": True,
37-
"ssl_mode": "verify-full",
38-
"ssl_certificate_authority": "./ssl/root.crt",
39-
"ssl_client_certificate": "./ssl/cert.crt",
40-
"ssl_client_private_key": "./ssl/pkey.key",
41-
"add_record_metadata": True,
42-
"hard_delete": False,
43-
"default_target_schema": "melty",
44-
}
4531

32+
# The below syntax is documented at https://docs.pytest.org/en/stable/deprecations.html#calling-fixtures-directly
33+
@pytest.fixture(scope="session", name="postgres_config")
34+
def postgres_config_fixture():
35+
return postgres_config()
4636

47-
@pytest.fixture(scope="session")
48-
def postgres_config_no_ssl():
49-
return {
50-
"dialect+driver": "postgresql+psycopg2",
51-
"host": "localhost",
52-
"user": "postgres",
53-
"password": "postgres",
54-
"database": "postgres",
55-
"port": 5433,
56-
"add_record_metadata": True,
57-
"hard_delete": False,
58-
"default_target_schema": "melty",
59-
}
6037

38+
@pytest.fixture(scope="session", name="postgres_config_no_ssl")
39+
def postgres_config_no_ssl_fixture():
40+
return postgres_config_no_ssl()
6141

62-
@pytest.fixture(scope="session")
63-
def postgres_config_ssh_tunnel():
64-
return {
65-
"sqlalchemy_url": "postgresql://postgres:[email protected]:5432/main",
66-
"ssh_tunnel": {
67-
"enable": True,
68-
"host": "127.0.0.1",
69-
"port": 2223,
70-
"username": "melty",
71-
"private_key": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn\nNhAAAAAwEAAQAAAYEAvIGU0pRpThhIcaSPrg2+v7cXl+QcG0icb45hfD44yrCoXkpJp7nh\nHv0ObZL2Y1cG7eeayYF4AqD3kwQ7W89GN6YO9b/mkJgawk0/YLUyojTS9dbcTbdkfPzyUa\nvTMDjly+PIjfiWOEnUgPf1y3xONLkJU0ILyTmgTzSIMNdKngtdCGfytBCuNiPKU8hEdEVt\n82ebqgtLoSYn9cUcVVz6LewzUh8+YtoPb8Z/BIVEzU37HiE9MOYIBXjo1AEJSnOCkjwlVl\nPzLhcXKTPht0iwv/KnZNNg0LDmnU/z0n+nPq/EMflum8jRYbgp0C5hksPdc8e0eEKd9gak\nt7B0ta3Mjt5b8HPQdBGZI/QFufEnSOxfJmoK4Bvjy/oUwE0hGU6po5g+4T2j6Bqqm2I+yV\nEbkP/UiuD/kEiT0C3yCV547gIDjN2ME9tGJDkd023BFvqn3stFVVZ5WsisRKGc+lvTfqeA\nJyKFaVt5a23y68ztjEMVrMLksRuEF8gG5kV7EGyjAAAFiCzGBRksxgUZAAAAB3NzaC1yc2\nEAAAGBALyBlNKUaU4YSHGkj64Nvr+3F5fkHBtInG+OYXw+OMqwqF5KSae54R79Dm2S9mNX\nBu3nmsmBeAKg95MEO1vPRjemDvW/5pCYGsJNP2C1MqI00vXW3E23ZHz88lGr0zA45cvjyI\n34ljhJ1ID39ct8TjS5CVNCC8k5oE80iDDXSp4LXQhn8rQQrjYjylPIRHRFbfNnm6oLS6Em\nJ/XFHFVc+i3sM1IfPmLaD2/GfwSFRM1N+x4hPTDmCAV46NQBCUpzgpI8JVZT8y4XFykz4b\ndIsL/yp2TTYNCw5p1P89J/pz6vxDH5bpvI0WG4KdAuYZLD3XPHtHhCnfYGpLewdLWtzI7e\nW/Bz0HQRmSP0BbnxJ0jsXyZqCuAb48v6FMBNIRlOqaOYPuE9o+gaqptiPslRG5D/1Irg/5\nBIk9At8gleeO4CA4zdjBPbRiQ5HdNtwRb6p97LRVVWeVrIrEShnPpb036ngCcihWlbeWtt\n8uvM7YxDFazC5LEbhBfIBuZFexBsowAAAAMBAAEAAAGAflHjdb2oV4HkQetBsSRa18QM1m\ncxAoOE+SiTYRudGQ6KtSzY8MGZ/xca7QiXfXhbF1+llTTiQ/i0Dtu+H0blyfLIgZwIGIsl\nG2GCf/7MoG//kmhaFuY3O56Rj3MyQVVPgHLy+VhE6hFniske+C4jhicc/aL7nOu15n3Qad\nJLmV8KB9EIjevDoloXgk9ot/WyuXKLmMaa9rFIA+UDmJyGtfFbbsOrHbj8sS11/oSD14RT\nLBygEb2EUI52j2LmY/LEvUL+59oCuJ6Y/h+pMdFeuHJzGjrVb573KnGwejzY24HHzzebrC\nQ+9NyVCTyizPHNu9w52/GPEZQFQBi7o9cDMd3ITZEPIaIvDHsUwPXaHUBHy/XHQTs8pDqk\nzCMcAs5zdzao2I0LQ+ZFYyvl1rue82ITjDISX1WK6nFYLBVXugi0rLGEdH6P+Psfl3uCIf\naW7c12/BpZz2Pql5AuO1wsu4rmz2th68vaC/0IDqWekIbW9qihFbqnhfAxRsIURjpBAAAA\nwDhIQPsj9T9Vud3Z/TZjiAKCPbg3zi082u1GMMxXnNQtKO3J35wU7VUcAxAzosWr+emMqS\nU0qW+a5RXr3sqUOqH85b5+Xw0yv2sTr2pL0ALFW7Tq1mesCc3K0So3Yo30pWRIOxYM9ihm\nE4ci/3mN5kcKWwvLLomFPRU9u0XtIGKnF/cNByTuz9fceR6Pi6mQXZawv+OOMiBeu0gbyp\nF1uVe8PCshzCrWTE3UjRpQxy9gizvSbGZyGQi1Lm42JXKG3wAAAMEA4r4CLM1xsyxBBMld\nrxiTqy6bfrZjKkT5MPjBjp+57i5kW9NVqGCnIy/m98pLTuKjTCDmUuWQXS+oqhHw5vq/wj\nRvQYqkJDz1UGmC1lD2qyqERjOiWa8/iy4dXSLeHCT70+/xR2dBb0z8cT++yZEqLdEZSnHG\nyRaZMHot1OohVDqJS8nEbxOzgPGdopRMiX6ws/p5/k9YAGkHx0hszA8cn/Tk2/mdS5lugw\nY7mdXzfcKvxkgoFrG7XowqRVrozcvDAAAAwQDU1ITasquNLaQhKNqiHx/N7bvKVO33icAx\nNdShqJEWx/g9idvQ25sA1Ubc1a+Ot5Lgfrs2OBKe+LgSmPAZOjv4ShqBHtsSh3am8/K1xR\ngQKgojLL4FhtgxtwoZrVvovZHGV3g2A28BRGbKIGVGPsOszJALU7jlLlcTHlB7SCQBI8FQ\nvTi2UEsfTmA22NnuVPITeqbmAQQXkSZcZbpbvdc0vQzp/3iOb/OCrIMET3HqVEMyQVsVs6\nxa9026AMTGLaEAAAATcm9vdEBvcGVuc3NoLXNlcnZlcg==\n-----END OPENSSH PRIVATE KEY-----", # noqa: E501
72-
},
73-
}
42+
43+
@pytest.fixture(scope="session", name="postgres_config_ssh_tunnel")
44+
def postgres_config_ssh_tunnel_fixture():
45+
return postgres_config_ssh_tunnel()
7446

7547

7648
@pytest.fixture
7749
def postgres_target(postgres_config) -> TargetPostgres:
7850
return TargetPostgres(config=postgres_config)
7951

8052

81-
def create_engine(target_postgres: TargetPostgres) -> sqlalchemy.engine.Engine:
82-
return TargetPostgres.default_sink_class.connector_class(
83-
config=target_postgres.config
84-
)._engine
85-
86-
8753
def singer_file_to_target(file_name, target) -> None:
8854
"""Singer file to Target, emulates a tap run
8955

0 commit comments

Comments
 (0)