Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Functional test for hourly microbatch model #11220

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Functional test for hourly microbatch model #11220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1500222
Functional test for hourly microbatch model
dbeatty10 Jan 16, 2025
5f2c032
Use today's date for functional test for hourly microbatch model
dbeatty10 Jan 16, 2025
3804e04
Use today's date for functional test for hourly microbatch model
dbeatty10 Jan 16, 2025
a669c1d
Restore to original
dbeatty10 Jan 17, 2025
85e0e32
Only use alphanumeric characters within batch ids
dbeatty10 Jan 17, 2025
fe73096
Add tests for batch_id and change expected output for format_batch_start
dbeatty10 Jan 17, 2025
65a1db0
Handle missing batch_start
dbeatty10 Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions core/dbt/materializations/incremental/microbatch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,13 +194,21 @@

@staticmethod
def batch_id(start_time: datetime, batch_size: BatchSize) -> str:
return MicrobatchBuilder.format_batch_start(start_time, batch_size).replace("-", "")
return MicrobatchBuilder.format_batch_start(start_time, batch_size)

Check warning on line 197 in core/dbt/materializations/incremental/microbatch.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/materializations/incremental/microbatch.py#L197 

Added line #L197 was not covered by tests

@staticmethod
def format_batch_start(batch_start: datetime, batch_size: BatchSize) -> str:
return str(
batch_start.date() if (batch_start and batch_size != BatchSize.hour) else batch_start
)
"""Format the passed in datetime based on the batch_size.

2024-09-17 16:06:00 + Batchsize.day -> 20240917
2024-09-17 16:06:00 + Batchsize.hour -> 20240917T16
"""
# If we want a date only
if batch_size != BatchSize.hour:
return batch_start.strftime("%Y%m%d")

Check warning on line 208 in core/dbt/materializations/incremental/microbatch.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/materializations/incremental/microbatch.py#L207-L208 

Added lines #L207 - L208 were not covered by tests

# If we want date + time
return batch_start.strftime("%Y%m%dT%H")

Check warning on line 211 in core/dbt/materializations/incremental/microbatch.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/materializations/incremental/microbatch.py#L211 

Added line #L211 was not covered by tests

@staticmethod
def ceiling_timestamp(timestamp: datetime, batch_size: BatchSize) -> datetime:
Expand Down
22 changes: 18 additions & 4 deletions tests/unit/materializations/incremental/test_microbatch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,13 +602,27 @@ def test_offset_timestamp(self, timestamp, batch_size, offset, expected_timestam
def test_truncate_timestamp(self, timestamp, batch_size, expected_timestamp):
assert MicrobatchBuilder.truncate_timestamp(timestamp, batch_size) == expected_timestamp

@pytest.mark.parametrize(
"batch_size,start_time,expected_formatted_start_time",
[
(BatchSize.year, None, ""),
(BatchSize.year, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.month, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.day, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.hour, datetime(2020, 1, 1, 1), "20200101T01"),
],
)
def test_batch_id(self, batch_size, start_time, expected_formatted_start_time):
assert MicrobatchBuilder.batch_id(start_time, batch_size) == expected_formatted_start_time

@pytest.mark.parametrize(
"batch_size,batch_start,expected_formatted_batch_start",
[
(BatchSize.year, datetime(2020, 1, 1, 1), "2020-01-01"),
(BatchSize.month, datetime(2020, 1, 1, 1), "2020-01-01"),
(BatchSize.day, datetime(2020, 1, 1, 1), "2020-01-01"),
(BatchSize.hour, datetime(2020, 1, 1, 1), "2020-01-01 01:00:00"),
(BatchSize.year, None, ""),
(BatchSize.year, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.month, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.day, datetime(2020, 1, 1, 1), "20200101"),
(BatchSize.hour, datetime(2020, 1, 1, 1), "20200101T01"),
],
)
def test_format_batch_start(self, batch_size, batch_start, expected_formatted_batch_start):
Expand Down
Loading