Skip to content

Remove More Event Logging #2353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexrichey merged 1 commit into from
May 6, 2026
Merged

Remove More Event Logging #2353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 2 additions & 90 deletions dcpy/connectors/edm/publishing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,6 @@
from dcpy.configuration import (
BUILD_NAME,
CI,
DEV_FLAG,
IGNORED_LOGGING_BUILDS,
LOGGING_DB,
LOGGING_SCHEMA,
LOGGING_TABLE_NAME,
PRODUCTS_TO_LOG,
PUBLISHING_BUCKET,
)
from dcpy.connectors.registry import GenericConnector, VersionedConnector
Expand All @@ -32,8 +26,8 @@
ProductKey,
PublishKey,
)
from dcpy.models.lifecycle.builds import BuildMetadata, EventLog, EventType
from dcpy.utils import git, metadata, postgres, s3, versions
from dcpy.models.lifecycle.builds import BuildMetadata
from dcpy.utils import git, s3, versions
from dcpy.utils.logging import logger


Expand Down Expand Up @@ -283,24 +277,6 @@ def upload_build(
logger.info(f'Uploading {output_path} to {build_key.path} with ACL "{acl}"')
upload(output_path, build_key, acl=acl, max_files=max_files)

if (
build_key.build not in IGNORED_LOGGING_BUILDS
and build_key.product in PRODUCTS_TO_LOG
):
version = get_version(build_key)
run_details = metadata.get_run_details()
event_metadata = EventLog(
event=EventType.BUILD,
product=build_key.product,
version=version,
path=build_key.path,
old_path=None,
timestamp=run_details.timestamp,
runner_type=run_details.type,
runner=run_details.runner_string,
)
# log_event_in_db(event_metadata)

return build_key


Expand Down Expand Up @@ -357,19 +333,6 @@ def promote_to_draft(
)
logger.info(f"Downloaded build_metadata.json from {draft_key.path}")

run_details = metadata.get_run_details()
event_metadata = EventLog(
event=EventType.PROMOTE_TO_DRAFT,
product=draft_key.product,
version=draft_key.version,
path=draft_key.path,
old_path=build_key.path,
timestamp=run_details.timestamp,
runner_type=run_details.type,
runner=run_details.runner_string,
)
# log_event_in_db(event_metadata)

return draft_key


Expand Down Expand Up @@ -485,19 +448,6 @@ def publish(
)
logger.info(f"Downloaded build_metadata.json from {publish_key.path}")

run_details = metadata.get_run_details()
event_metadata = EventLog(
event=EventType.PUBLISH,
product=publish_key.product,
version=draft_key.version, # this is release version, not patched
path=publish_key.path,
old_path=draft_key.path,
timestamp=run_details.timestamp,
runner_type=run_details.type,
runner=run_details.runner_string,
)
# log_event_in_db(event_metadata)

return publish_key


Expand Down Expand Up @@ -718,44 +668,6 @@ def download_gis_dataset(dataset_name: str, version: str, target_folder: Path):
return file_path


def log_event_in_db(event_details: EventLog) -> None:
"""
Logs event metadata to a PostgreSQL database if the product is in the approved list
of products and not in a development environment. Otherwise it skips logging.
"""

if event_details.product not in PRODUCTS_TO_LOG:
logger.warn(
f"❗️ Product {event_details.product} not on the list of products to log in db. Skipping event metadata logging..."
)
return
if DEV_FLAG:
logger.info("DEV_FLAG env var found, skipping event metadata logging")
return
logger.info(
f"Logging event '{event_details.event}' metadata for product {event_details.product} in db..."
)
pg_client = postgres.PostgresClient(database=LOGGING_DB, schema=LOGGING_SCHEMA)
query = f"""
INSERT INTO {LOGGING_SCHEMA}.{LOGGING_TABLE_NAME}
(product, version, event, path, old_path, timestamp, runner_type, runner, custom_fields)
VALUES
(:product, :version, :event, :path, :old_path, :timestamp, :runner_type, :runner, :custom_fields)
"""
pg_client.execute_query(
query,
product=event_details.product,
version=event_details.version,
event=event_details.event.value,
path=event_details.path,
old_path=event_details.old_path,
timestamp=event_details.timestamp,
runner_type=event_details.runner_type,
runner=event_details.runner,
custom_fields=json.dumps(event_details.custom_fields),
)


class PublishedConnector(VersionedConnector):
conn_type: str = "edm.publishing.published"

Expand Down
99 changes: 0 additions & 99 deletions dcpy/test/connectors/edm/test_publishing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import json
import os
from datetime import datetime
from pathlib import Path
Expand Down Expand Up @@ -169,40 +168,6 @@ def test_upload_build_file_not_found(create_buckets, mock_data_constants):
)


@patch("dcpy.connectors.edm.publishing.log_event_in_db")
def test_upload_build_validate_logging_logic(
mock_log_event,
create_buckets,
create_temp_filesystem,
mock_data_constants,
):
data_path = mock_data_constants["TEST_DATA_DIR"]
build_to_ignore = publishing.IGNORED_LOGGING_BUILDS[0]
publishing.upload_build(
data_path,
product=TEST_PRODUCT_NAME,
build=build_to_ignore,
acl=TEST_ACL,
)
# Ensure log_event_in_db is not called since the build is ignored
mock_log_event.assert_not_called()

mock_log_event.reset_mock()

non_ignored_build = TEST_BUILD
non_ignored_product = "db-template"
assert non_ignored_product in publishing.PRODUCTS_TO_LOG # sanity check
assert non_ignored_build not in publishing.IGNORED_LOGGING_BUILDS # sanity check
publishing.upload_build(
data_path,
product=non_ignored_product,
build=non_ignored_build,
acl=TEST_ACL,
)
# Ensure log_event_in_db is called
mock_log_event.assert_called_once()


def test_publish_patch(create_buckets, create_temp_filesystem, mock_data_constants):
"""
Tests publish function when a version already exists. When is_path = True,
Expand Down Expand Up @@ -615,67 +580,3 @@ def test_validate_or_patch_version_version_already_exists(get_published_versions
version=version_to_patch,
is_patch=False,
)


@pytest.fixture(scope="function")
def mock_event_log():
run_details = publishing.metadata.get_run_details()
return publishing.EventLog(
product=TEST_PRODUCT_NAME,
version=TEST_VERSION,
event=publishing.EventType.BUILD,
path="/new/path",
old_path="/old/path",
timestamp=run_details.timestamp,
runner_type=run_details.type,
runner=run_details.runner_string,
custom_fields={"key": "value"},
)


@patch("dcpy.connectors.edm.publishing.postgres.PostgresClient")
def test_log_event_skipped_conditions(mock_db_client, mock_event_log):
"""
Test the `log_event_in_db` function to ensure that no database query is executed
when the product is not in the allowed list or when the DEV_FLAG is set to True.
"""
mock_db_client_instance = mock_db_client.return_value
with patch("dcpy.connectors.edm.publishing.DEV_FLAG", False):
assert mock_event_log.product not in publishing.PRODUCTS_TO_LOG # sanity check
publishing.log_event_in_db(mock_event_log)
mock_db_client_instance.execute_query.assert_not_called()

with patch("dcpy.connectors.edm.publishing.DEV_FLAG", True):
mock_event_log.product = "db-template"
assert mock_event_log.product in publishing.PRODUCTS_TO_LOG # sanity check
publishing.log_event_in_db(mock_event_log)
mock_db_client_instance.execute_query.assert_not_called()


@patch("dcpy.connectors.edm.publishing.postgres.PostgresClient")
def test_log_event_success(mock_db_client, mock_event_log):
mock_db_client_instance = mock_db_client.return_value

mock_event_log.product = "db-template"
assert mock_event_log.product in publishing.PRODUCTS_TO_LOG # sanity check

publishing.log_event_in_db(mock_event_log)

query = f"""
INSERT INTO {publishing.LOGGING_SCHEMA}.{publishing.LOGGING_TABLE_NAME}
(product, version, event, path, old_path, timestamp, runner_type, runner, custom_fields)
VALUES
(:product, :version, :event, :path, :old_path, :timestamp, :runner_type, :runner, :custom_fields)
"""
mock_db_client_instance.execute_query.assert_called_once_with(
query,
product=mock_event_log.product,
version=mock_event_log.version,
event=mock_event_log.event.value,
path=mock_event_log.path,
old_path=mock_event_log.old_path,
timestamp=mock_event_log.timestamp,
runner_type=mock_event_log.runner_type,
runner=mock_event_log.runner,
custom_fields=json.dumps(mock_event_log.custom_fields),
)
Loading