From dab157078f8c82fe118ea767f7bf789397663925 Mon Sep 17 00:00:00 2001 From: KoomeKiriinya Date: Mon, 28 Jul 2025 14:55:54 +0300 Subject: [PATCH] fix pre-commit in modules/gcp/bigquery --- modules/gcp/bigquery/.tflint.hcl | 1 + modules/gcp/bigquery/main.tf | 88 +++++++++++++++---------------- modules/gcp/bigquery/outputs.tf | 0 modules/gcp/bigquery/variables.tf | 3 +- 4 files changed, 45 insertions(+), 47 deletions(-) create mode 120000 modules/gcp/bigquery/.tflint.hcl create mode 100644 modules/gcp/bigquery/outputs.tf diff --git a/modules/gcp/bigquery/.tflint.hcl b/modules/gcp/bigquery/.tflint.hcl new file mode 120000 index 0000000..a5b83c7 --- /dev/null +++ b/modules/gcp/bigquery/.tflint.hcl @@ -0,0 +1 @@ +../../../.tflint.hcl \ No newline at end of file diff --git a/modules/gcp/bigquery/main.tf b/modules/gcp/bigquery/main.tf index fe2c43e..608f266 100644 --- a/modules/gcp/bigquery/main.tf +++ b/modules/gcp/bigquery/main.tf @@ -1,22 +1,22 @@ locals { - + bq_tables = flatten([ for dataset_key, values in var.bq_datasets : [ - for table_key, table_values in lookup(values, "tables") : { - dataset_id = dataset_key - table_id = table_key + for table_key, table_values in lookup(values, "tables", {}) : { + dataset_id = dataset_key + table_id = table_key deletion_protection = try(table_values.deletion_protection, false) - friendly_name = try(table_values.friendly_name, table_key) - description = try(table_values.description, null) - source_uris = table_values.source_uris - source_format = try(table_values.source_format, "PARQUET") - } - ] - ]) + friendly_name = try(table_values.friendly_name, table_key) + description = try(table_values.description, null) + source_uris = table_values.source_uris + source_format = try(table_values.source_format, "PARQUET") + } + ] + ]) bq_datasets_access_policy = flatten([ for dataset_key, values in var.bq_datasets : [ - for role_key, members in lookup(values, "access") : { + for role_key, members in lookup(values, "access", {}) : { dataset_id = dataset_key role = role_key members = members @@ -26,19 +26,15 @@ locals { labels = var.labels - postgres_password = var.postgres_password - - stored_procedures = {} - cloudsql_scheduled_postgres_transfers = {} } -resource "google_project_service" "bigquerydatatransfer"{ +resource "google_project_service" "bigquerydatatransfer" { project = var.project service = "bigquerydatatransfer.googleapis.com" } -resource "google_project_service" "bigquery"{ +resource "google_project_service" "bigquery" { project = var.project service = "bigquery.googleapis.com" } @@ -51,9 +47,9 @@ resource "google_bigquery_dataset" "bq_datasets" { max_time_travel_hours = 168 # default_table_expiration_ms = 3600000 # 1 hr - friendly_name = each.key - description = each.value.dataset_description - labels = local.labels + friendly_name = each.key + description = each.value.dataset_description + labels = local.labels delete_contents_on_destroy = each.value.force_destroy } @@ -71,50 +67,50 @@ resource "google_bigquery_dataset_iam_binding" "bq_access" { resource "google_bigquery_data_transfer_config" "cloudsql_postgres_transfer" { for_each = { - for dt_config in var.cloudsql_scheduled_postgres_transfers: dt_config.name => dt_config + for dt_config in var.cloudsql_scheduled_postgres_transfers : dt_config.name => dt_config } - display_name = each.key - project = var.project - location = "US" - data_source_id = "postgresql" - schedule = each.value.schedule + display_name = each.key + project = var.project + location = "US" + data_source_id = "postgresql" + schedule = each.value.schedule destination_dataset_id = google_bigquery_dataset.bq_datasets[each.value.destination_dataset].dataset_id params = { - "assets": jsonencode(each.value.source_table_names) - "connector.authentication.username": each.value.username - "connector.authentication.password": var.postgres_password - "connector.database": each.value.database - "connector.endpoint.host": each.value.host - "connector.endpoint.port": 5432 - "connector.encryptionMode": each.value.encryption_mode - "connector.networkAttachment": try(each.value.network_attachment, null) - "connector.schema": each.value.schema - + "assets" : jsonencode(each.value.source_table_names) + "connector.authentication.username" : each.value.username + "connector.authentication.password" : var.postgres_password + "connector.database" : each.value.database + "connector.endpoint.host" : each.value.host + "connector.endpoint.port" : 5432 + "connector.encryptionMode" : each.value.encryption_mode + "connector.networkAttachment" : try(each.value.network_attachment, null) + "connector.schema" : each.value.schema + } service_account_name = var.service_account_email - depends_on = [google_bigquery_dataset.bq_datasets] + depends_on = [google_bigquery_dataset.bq_datasets] } resource "google_bigquery_table" "bq_tables" { depends_on = [google_bigquery_dataset.bq_datasets] - project = var.project + project = var.project - for_each = { - for table in local.bq_tables: table.table_id => table + for_each = { + for table in local.bq_tables : table.table_id => table } table_id = each.key dataset_id = each.value.dataset_id deletion_protection = each.value.deletion_protection - friendly_name = each.value.friendly_name - description = try(each.value.description, null) + friendly_name = each.value.friendly_name + description = try(each.value.description, null) external_data_configuration { - autodetect = true # Parquet files used + autodetect = true # Parquet files used source_format = each.value.source_format - source_uris = each.value.source_uris + source_uris = each.value.source_uris } labels = local.labels @@ -124,4 +120,4 @@ resource "google_bigquery_table" "bq_tables" { external_data_configuration ] } -} \ No newline at end of file +} diff --git a/modules/gcp/bigquery/outputs.tf b/modules/gcp/bigquery/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/gcp/bigquery/variables.tf b/modules/gcp/bigquery/variables.tf index 64c24f1..57f5fd7 100644 --- a/modules/gcp/bigquery/variables.tf +++ b/modules/gcp/bigquery/variables.tf @@ -27,4 +27,5 @@ variable "postgres_password" { variable "labels" { type = map(any) -} \ No newline at end of file + description = "Labels on the bigquery dataset" +}