fix(ingest): postgres - ignore information_schema tables by default (d…

…atahub-project#4069)
njalan · Feb 10, 2022 · 9bdc9af · 9bdc9af
1 parent 076848f
commit 9bdc9af
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 6 deletions.
diff --git a/metadata-ingestion/source_docs/postgres.md b/metadata-ingestion/source_docs/postgres.md
@@ -16,10 +16,10 @@ This plugin extracts the following:
 - database_alias (optional) can be used to change the name of database to be ingested
 - Table, row, and column statistics via optional [SQL profiling](./sql_profiles.md)
 
-| Capability        | Status | Details                                  | 
-|-------------------|--------|------------------------------------------|
-| Data Containers   | ✔️     |                                          |
-| Data Domains      | ✔️     | [link](../../docs/domains.md)            |
+| Capability      | Status | Details                       |
+| --------------- | ------ | ----------------------------- |
+| Data Containers | ✔️     |                               |
+| Data Domains    | ✔️     | [link](../../docs/domains.md) |
 
 ## Quickstart recipe
 
@@ -53,10 +53,10 @@ Note that a `.` is used to denote nested fields in the YAML recipe.
 As a SQL-based service, the Athena integration is also supported by our SQL profiler. See [here](./sql_profiles.md) for more details on configuration.
 
 | Field                          | Required | Default  | Description                                                                                                                                                                             |
-|--------------------------------|----------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ------------------------------ | -------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `username`                     |          |          | PostgreSQL username.                                                                                                                                                                    |
 | `password`                     |          |          | PostgreSQL password.                                                                                                                                                                    |
-| `host_port`                    | ✅        |          | PostgreSQL host URL.                                                                                                                                                                    |
+| `host_port`                    | ✅       |          | PostgreSQL host URL.                                                                                                                                                                    |
 | `database`                     |          |          | PostgreSQL database.                                                                                                                                                                    |
 | `database_alias`               |          |          | Alias to apply to database when ingesting.                                                                                                                                              |
 | `env`                          |          | `"PROD"` | Environment to use in namespace when constructing URNs.                                                                                                                                 |

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py b/metadata-ingestion/src/datahub/ingestion/source/sql/postgres.py
@@ -9,6 +9,7 @@
 # https://geoalchemy-2.readthedocs.io/en/latest/core_tutorial.html#reflecting-tables.
 from geoalchemy2 import Geometry  # noqa: F401
 
+from datahub.configuration.common import AllowDenyPattern
 from datahub.ingestion.source.sql.sql_common import (
     BasicSQLAlchemyConfig,
     SQLAlchemySource,
@@ -29,6 +30,7 @@
 class PostgresConfig(BasicSQLAlchemyConfig):
     # defaults
     scheme = "postgresql+psycopg2"
+    schema_pattern = AllowDenyPattern(deny=["information_schema"])
 
     def get_identifier(self: BasicSQLAlchemyConfig, schema: str, table: str) -> str:
         regular = f"{schema}.{table}"