Source versioning: extend tests

Author: bobbyiliev

integration/mysql/mysql_bootstrap.sql

@@ -9,6 +9,7 @@ GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT, LOC
 
 FLUSH PRIVILEGES;
 
+-- Basic tables
 CREATE TABLE IF NOT EXISTS mysql_table1
 (
     id INT AUTO_INCREMENT PRIMARY KEY,
@@ -31,14 +32,128 @@ CREATE TABLE IF NOT EXISTS mysql_table3
     id INT AUTO_INCREMENT PRIMARY KEY
 );
 
+-- Table with ENUM type (needs text_columns)
 CREATE TABLE IF NOT EXISTS mysql_table4
 (
     id INT AUTO_INCREMENT PRIMARY KEY,
     status ENUM('active', 'inactive', 'deleted') NOT NULL DEFAULT 'active'
 );
 
+-- Table with various data types
+CREATE TABLE IF NOT EXISTS mysql_table5
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    name VARCHAR(255),
+    email VARCHAR(255),
+    age INT,
+    salary DECIMAL(10, 2),
+    is_active BOOLEAN,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP
+);
+
+-- Table with SET type (needs text_columns)
+CREATE TABLE IF NOT EXISTS mysql_table6
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    tags SET('tag1', 'tag2', 'tag3', 'tag4') DEFAULT NULL,
+    status ENUM('pending', 'approved', 'rejected') DEFAULT 'pending'
+);
+
+-- Table with special characters in names
+CREATE TABLE IF NOT EXISTS mysql_table7
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    `name_with_underscores` VARCHAR(255),
+    `name-with-dashes` VARCHAR(255),
+    description TEXT,
+    binary_data BLOB
+);
+
+-- Table with nullable columns
+CREATE TABLE IF NOT EXISTS mysql_table8
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    nullable_string VARCHAR(255) NULL,
+    nullable_int INT NULL,
+    nullable_timestamp TIMESTAMP NULL,
+    required_field VARCHAR(255) NOT NULL
+);
+
+-- Table with numeric types
+CREATE TABLE IF NOT EXISTS mysql_table9
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    tiny_int TINYINT,
+    small_int SMALLINT,
+    medium_int MEDIUMINT,
+    big_int BIGINT,
+    float_val FLOAT,
+    double_val DOUBLE,
+    decimal_val DECIMAL(10, 2)
+);
+
+-- Table with date/time types
+CREATE TABLE IF NOT EXISTS mysql_table10
+(
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    date_col DATE,
+    time_col TIME,
+    datetime_col DATETIME,
+    timestamp_col TIMESTAMP,
+    year_col YEAR
+);
+
 -- Insert sample data
-INSERT INTO mysql_table1 (name, about, banned) VALUES ('John Doe', 'Lorem ipsum dolor sit amet', 0), ('Jane Doe', 'Lorem ipsum dolor sit amet', 1), ('Alice', 'Lorem ipsum dolor sit amet', 0), ('Bob', 'Lorem ipsum dolor sit amet', 1), ('Charlie', 'Lorem ipsum dolor sit amet', 0);
-INSERT INTO mysql_table2 (id, updated_at) VALUES (1, NOW()), (2, NOW()), (3, NOW()), (4, NOW()), (5, NOW());
-INSERT INTO mysql_table2 (id) VALUES (1), (2), (3), (4), (5);
+INSERT INTO mysql_table1 (name, about, banned) VALUES 
+    ('John Doe', 'Lorem ipsum dolor sit amet', 0), 
+    ('Jane Doe', 'Lorem ipsum dolor sit amet', 1), 
+    ('Alice', 'Lorem ipsum dolor sit amet', 0), 
+    ('Bob', 'Lorem ipsum dolor sit amet', 1), 
+    ('Charlie', 'Lorem ipsum dolor sit amet', 0);
+
+INSERT INTO mysql_table2 (id, name, about, banned, updated_at) VALUES 
+    (1, 'Record 1', 'First record', 0, NOW()), 
+    (2, 'Record 2', 'Second record', 1, NOW()), 
+    (3, 'Record 3', 'Third record', 0, NOW()), 
+    (4, 'Record 4', 'Fourth record', 1, NOW()), 
+    (5, 'Record 5', 'Fifth record', 0, NOW());
+
 INSERT INTO mysql_table3 (id) VALUES (NULL), (NULL), (NULL), (NULL), (NULL);
+
+INSERT INTO mysql_table4 (status) VALUES 
+    ('active'), ('inactive'), ('deleted'), ('active'), ('inactive');
+
+INSERT INTO mysql_table5 (name, email, age, salary, is_active, created_at) VALUES
+    ('John Doe', '[email protected]', 30, 50000.00, 1, NOW()),
+    ('Jane Smith', '[email protected]', 25, 60000.00, 1, NOW()),
+    ('Bob Johnson', '[email protected]', 35, 55000.00, 0, NOW()),
+    ('Alice Brown', '[email protected]', 28, 65000.00, 1, NOW()),
+    ('Charlie Wilson', '[email protected]', 32, 70000.00, 1, NOW());
+
+INSERT INTO mysql_table6 (tags, status) VALUES
+    ('tag1,tag2', 'pending'),
+    ('tag3', 'approved'),
+    ('tag1,tag3,tag4', 'rejected'),
+    (NULL, 'pending'),
+    ('tag2', 'approved');
+
+INSERT INTO mysql_table7 (`name_with_underscores`, `name-with-dashes`, description, binary_data) VALUES
+    ('underscore_name', 'dash-name', 'Description with special chars: !@#$%', UNHEX('DEADBEEF')),
+    ('another_underscore', 'another-dash', 'More special chars: &*()', UNHEX('CAFEBABE'));
+
+INSERT INTO mysql_table8 (nullable_string, nullable_int, nullable_timestamp, required_field) VALUES
+    ('Has value', 100, NOW(), 'Required'),
+    (NULL, NULL, NULL, 'Required'),
+    ('Another value', 200, NOW(), 'Required'),
+    (NULL, 300, NULL, 'Required');
+
+INSERT INTO mysql_table9 (tiny_int, small_int, medium_int, big_int, float_val, double_val, decimal_val) VALUES
+    (127, 32767, 8388607, 9223372036854775807, 3.14159, 2.71828, 1234.56),
+    (-128, -32768, -8388608, -9223372036854775808, -3.14159, -2.71828, -1234.56),
+    (0, 0, 0, 0, 0.0, 0.0, 0.00);
+
+INSERT INTO mysql_table10 (date_col, time_col, datetime_col, timestamp_col, year_col) VALUES
+    ('2024-01-01', '12:00:00', '2024-01-01 12:00:00', NOW(), 2024),
+    ('2024-06-15', '18:30:00', '2024-06-15 18:30:00', NOW(), 2024),
+    ('2024-12-31', '23:59:59', '2024-12-31 23:59:59', NOW(), 2024);

integration/postgres/postgres_bootstrap.sql

@@ -1,6 +1,7 @@
 ALTER SYSTEM SET wal_level = logical;
 ALTER ROLE postgres WITH REPLICATION;
 
+-- Basic tables
 CREATE TABLE table1 (
     id INT GENERATED ALWAYS AS IDENTITY
 );
@@ -14,14 +15,96 @@ CREATE TABLE table3 (
     id INT GENERATED ALWAYS AS IDENTITY
 );
 
--- Enable REPLICA for both tables
+-- Table with various data types for testing
+CREATE TABLE table4 (
+    id INT PRIMARY KEY,
+    name VARCHAR(255),
+    email VARCHAR(255),
+    age INT,
+    salary DECIMAL(10, 2),
+    is_active BOOLEAN,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP
+);
+
+-- Table with unsupported types that need text_columns
+CREATE TABLE table5 (
+    id INT PRIMARY KEY,
+    data JSONB,
+    tags TEXT[],
+    metadata JSONB
+);
+
+-- Table with special characters and edge cases
+CREATE TABLE table6 (
+    id INT PRIMARY KEY,
+    name_with_dots VARCHAR(255),
+    "name-with-dashes" VARCHAR(255),
+    "name with spaces" VARCHAR(255),
+    description TEXT,
+    binary_data BYTEA
+);
+
+-- Table with nullable columns
+CREATE TABLE table7 (
+    id INT PRIMARY KEY,
+    nullable_string VARCHAR(255),
+    nullable_int INT,
+    nullable_timestamp TIMESTAMP,
+    required_field VARCHAR(255) NOT NULL
+);
+
+-- Table with numeric types
+CREATE TABLE table8 (
+    id SERIAL PRIMARY KEY,
+    small_int SMALLINT,
+    big_int BIGINT,
+    real_num REAL,
+    double_num DOUBLE PRECISION,
+    numeric_val NUMERIC(10, 2)
+);
+
+-- Enable REPLICA IDENTITY for all tables
 ALTER TABLE table1 REPLICA IDENTITY FULL;
 ALTER TABLE table2 REPLICA IDENTITY FULL;
 ALTER TABLE table3 REPLICA IDENTITY FULL;
+ALTER TABLE table4 REPLICA IDENTITY FULL;
+ALTER TABLE table5 REPLICA IDENTITY FULL;
+ALTER TABLE table6 REPLICA IDENTITY FULL;
+ALTER TABLE table7 REPLICA IDENTITY FULL;
+ALTER TABLE table8 REPLICA IDENTITY FULL;
 
--- Create publication on the created tables
-CREATE PUBLICATION mz_source FOR TABLE table1, table2, table3;
+-- Create publication on all tables
+CREATE PUBLICATION mz_source FOR TABLE table1, table2, table3, table4, table5, table6, table7, table8;
 
+-- Insert sample data
 INSERT INTO table1 VALUES (1), (2), (3), (4), (5);
 INSERT INTO table2 VALUES (1, NOW()), (2, NOW()), (3, NOW()), (4, NOW()), (5, NOW());
 INSERT INTO table3 VALUES (1), (2), (3), (4), (5);
+
+INSERT INTO table4 (id, name, email, age, salary, is_active, created_at) VALUES
+    (1, 'John Doe', '[email protected]', 30, 50000.00, true, NOW()),
+    (2, 'Jane Smith', '[email protected]', 25, 60000.00, true, NOW()),
+    (3, 'Bob Johnson', '[email protected]', 35, 55000.00, false, NOW()),
+    (4, 'Alice Brown', '[email protected]', 28, 65000.00, true, NOW()),
+    (5, 'Charlie Wilson', '[email protected]', 32, 70000.00, true, NOW());
+
+INSERT INTO table5 (id, data, tags, metadata) VALUES
+    (1, '{"key": "value", "number": 123}'::jsonb, ARRAY['tag1', 'tag2'], '{"meta": "data"}'::jsonb),
+    (2, '{"key": "value2", "number": 456}'::jsonb, ARRAY['tag3'], '{"meta": "data2"}'::jsonb),
+    (3, '{"key": "value3"}'::jsonb, ARRAY['tag1', 'tag3'], NULL::jsonb);
+
+INSERT INTO table6 (id, name_with_dots, "name-with-dashes", "name with spaces", description, binary_data) VALUES
+    (1, 'table.name', 'table-name', 'table name', 'Description with special chars: !@#$%', E'\\xDEADBEEF'),
+    (2, 'another.table', 'another-name', 'another name', 'More special chars: &*()', E'\\xCAFEBABE');
+
+INSERT INTO table7 (id, nullable_string, nullable_int, nullable_timestamp, required_field) VALUES
+    (1, 'Has value', 100, NOW(), 'Required'),
+    (2, NULL, NULL, NULL, 'Required'),
+    (3, 'Another value', 200, NOW(), 'Required'),
+    (4, NULL, 300, NULL, 'Required');
+
+INSERT INTO table8 (small_int, big_int, real_num, double_num, numeric_val) VALUES
+    (32767, 9223372036854775807, 3.14159, 2.71828, 1234.56),
+    (-32768, -9223372036854775808, -3.14159, -2.71828, -1234.56),
+    (0, 0, 0.0, 0.0, 0.00);

integration/sqlserver.tf

@@ -54,7 +54,7 @@ resource "materialize_source_sqlserver" "sqlserver_source_all" {
     database_name = materialize_connection_sqlserver.sqlserver_connection.database_name
   }
 
-  exclude_columns = ["dbo.table3.data", "dbo.table1.about", "dbo.table2.about"]
+  exclude_columns = ["dbo.table3.data", "dbo.table1.about", "dbo.table2.about", "dbo.table5.large_text", "dbo.table5.image_data", "dbo.table10.text_col"]
 }
 
 # SQL Server Source Table