source versioning: fix failing tests

Author: bobbyiliev

docs/resources/source_mysql.md

@@ -52,6 +52,7 @@ resource "materialize_source_mysql" "test" {
 
 ### Optional
 
+- `all_tables` (Boolean, Deprecated) Include all tables in the source. If `table` is specified, this will be ignored.
 - `cluster_name` (String) The cluster to maintain this source.
 - `comment` (String) Comment on an object in the database.
 - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set.

docs/resources/source_table_postgres.md

@@ -47,6 +47,7 @@ resource "materialize_source_table_postgres" "postgres_table_from_source" {
 
 - `comment` (String) Comment on an object in the database.
 - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set.
+- `exclude_columns` (List of String) Exclude specific columns when reading data from PostgreSQL.
 - `ownership_role` (String) The owernship role of the object.
 - `region` (String) The region to use for the resource connection. If not set, the default region is used.
 - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`.

integration/sqlserver.tf

@@ -54,7 +54,7 @@ resource "materialize_source_sqlserver" "sqlserver_source_all" {
     database_name = materialize_connection_sqlserver.sqlserver_connection.database_name
   }
 
-  exclude_columns = ["dbo.table3.data", "dbo.table1.about", "dbo.table2.about", "dbo.table5.large_text", "dbo.table5.image_data", "dbo.table5.xml_data", "dbo.table5.json_data", "dbo.table10.text_col", "dbo.table10.nvarchar_max"]
+  exclude_columns = ["dbo.table3.data", "dbo.table1.about", "dbo.table2.about", "dbo.table5.large_text", "dbo.table5.image_data", "dbo.table5.xml_data", "dbo.table5.json_data", "dbo.table6.description", "dbo.table6.binary_data", "dbo.table10.text_col", "dbo.table10.nvarchar_max"]
 }
 
 # SQL Server Source Table

pkg/materialize/source_mysql.go

@@ -15,6 +15,7 @@ type SourceMySQLBuilder struct {
 	ignoreColumns   []string
 	textColumns     []string
 	tables          []TableStruct
+	allTables       bool
 	exposeProgress  IdentifierSchemaStruct
 }
 
@@ -55,6 +56,11 @@ func (b *SourceMySQLBuilder) Tables(tables []TableStruct) *SourceMySQLBuilder {
 	return b
 }
 
+func (b *SourceMySQLBuilder) AllTables() *SourceMySQLBuilder {
+	b.allTables = true
+	return b
+}
+
 func (b *SourceMySQLBuilder) ExposeProgress(e IdentifierSchemaStruct) *SourceMySQLBuilder {
 	b.exposeProgress = e
 	return b
@@ -111,7 +117,9 @@ func (b *SourceMySQLBuilder) Create() error {
 		}
 		q.WriteString(`)`)
 	} else {
-		q.WriteString(` FOR ALL TABLES`)
+		if b.allTables {
+			q.WriteString(` FOR ALL TABLES`)
+		}
 	}
 
 	if b.exposeProgress.Name != "" {

pkg/materialize/source_mysql_test.go

@@ -21,6 +21,7 @@ func TestSourceMySQLAllTablesCreate(t *testing.T) {
 		).WillReturnResult(sqlmock.NewResult(1, 1))
 
 		b := NewSourceMySQLBuilder(db, sourceMySQL)
+		b.AllTables()
 		b.MySQLConnection(IdentifierSchemaStruct{Name: "mysql_connection", SchemaName: "schema", DatabaseName: "database"})
 
 		if err := b.Create(); err != nil {
@@ -37,6 +38,7 @@ func TestSourceMySQLSpecificTablesCreate(t *testing.T) {
 
 		b := NewSourceMySQLBuilder(db, sourceMySQL)
 		b.MySQLConnection(IdentifierSchemaStruct{Name: "mysql_connection", SchemaName: "schema", DatabaseName: "database"})
+		b.AllTables()
 		b.Tables([]TableStruct{
 			{
 				UpstreamName:       "table_1",

pkg/provider/acceptance_datasource_source_table_test.go

@@ -61,10 +61,6 @@ resource "materialize_source_postgres" "test" {
 		name = materialize_connection_postgres.postgres_connection.name
 	}
 	publication = "mz_source"
-	table {
-		upstream_name  = "table2"
-		upstream_schema_name = "public"
-	}
 }
 
 resource "materialize_source_table_postgres" "test" {

pkg/provider/acceptance_source_table_mysql_test.go

@@ -342,13 +342,7 @@ func testAccSourceTableMySQLWithENUMResource(nameSpace string) string {
 			name = materialize_connection_mysql.mysql_connection.name
 		}
 
-		table {
-			upstream_name        = "mysql_table6"
-			upstream_schema_name = "shop"
-			name                 = "mysql_table6_local"
-		}
-
-		text_columns = ["shop.mysql_table6.tags", "shop.mysql_table6.status"]
+		ignore_columns = ["shop.mysql_table10.year_col"]
 	}
 
 	resource "materialize_source_table_mysql" "test_enum" {
@@ -467,6 +461,8 @@ func testAccSourceTableMySQLWithDateTimeTypesResource(nameSpace string) string {
 
 		upstream_name         = "mysql_table10"
 		upstream_schema_name  = "shop"
+
+		exclude_columns = ["year_col"]
 	}
 	`, nameSpace)
 }

pkg/provider/acceptance_source_table_sqlserver_test.go

@@ -282,7 +282,7 @@ func TestAccSourceTableSQLServer_withUnsupportedTypes(t *testing.T) {
 					testAccCheckSourceTableSQLServerExists("materialize_source_table_sqlserver.test_unsupported"),
 					resource.TestCheckResourceAttr("materialize_source_table_sqlserver.test_unsupported", "name", nameSpace+"_table_unsupported"),
 					resource.TestCheckResourceAttr("materialize_source_table_sqlserver.test_unsupported", "upstream_name", "table5"),
-					resource.TestCheckResourceAttr("materialize_source_table_sqlserver.test_unsupported", "exclude_columns.#", "2"),
+					resource.TestCheckResourceAttr("materialize_source_table_sqlserver.test_unsupported", "exclude_columns.#", "4"),
 				),
 			},
 		},
@@ -396,7 +396,9 @@ func testAccSourceTableSQLServerWithUnsupportedTypesResource(nameSpace string) s
 
 		exclude_columns = [
 			"large_text",
-			"image_data"
+			"image_data",
+			"xml_data",
+			"json_data"
 		]
 	}
 	`, nameSpace)

pkg/resources/resource_source_mysql.go

@@ -79,6 +79,13 @@ var sourceMySQLSchema = map[string]*schema.Schema{
 			},
 		},
 	},
+	"all_tables": {
+		Description: "Include all tables in the source. If `table` is specified, this will be ignored.",
+		Deprecated:  "Use the new `materialize_source_table_mysql` resource instead.",
+		Type:        schema.TypeBool,
+		Optional:    true,
+		ForceNew:    true,
+	},
 	"expose_progress": IdentifierSchema(IdentifierSchemaParams{
 		Elem:        "expose_progress",
 		Description: "The name of the progress collection for the source. If this is not specified, the collection will be named `<src_name>_progress`.",
@@ -134,6 +141,10 @@ func sourceMySQLCreate(ctx context.Context, d *schema.ResourceData, meta any) di
 		b.Tables(t)
 	}
 
+	if v, ok := d.GetOk("all_tables"); ok && v.(bool) {
+		b.AllTables()
+	}
+
 	if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 {
 		columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{}))
 		if err != nil {

pkg/resources/resource_source_table_postgres_test.go

@@ -38,7 +38,7 @@ func TestResourceSourceTablePostgresCreate(t *testing.T) {
 		mock.ExpectExec(`CREATE TABLE "database"."schema"."table"
             FROM SOURCE "materialize"."public"."source"
             \(REFERENCE "upstream_schema"."upstream_table"\)
-            WITH \(TEXT COLUMNS \("column1", "column2"\)\);`).
+            WITH \(TEXT COLUMNS \("column1", "column2"\), EXCLUDE COLUMNS \("exclude1", "exclude2"\)\);`).
 			WillReturnResult(sqlmock.NewResult(1, 1))
 
 		// Query Id