[cherrypick][PLUGIN-1779] Add TRANSACTION_ISOLATION_LEVEL config in MySQL, PostgreSQL & SQL Server plugins to support 6.9.x release

.github/workflows/build-report.yml

@@ -21,6 +21,11 @@ on:
     types:
     - completed
 
+permissions:
+  actions: read    # Allows reading workflow run information
+  statuses: write  # Required if the action updates commit statuses
+  checks: write    # Required if it updates GitHub Checks API
+
 jobs:
   build:
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -37,12 +37,12 @@ jobs:
          && (github.event.action != 'labeled' || github.event.label.name == 'build')
          )
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.workflow_run.head_sha }}
 
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}

.github/workflows/e2e.yml

@@ -16,9 +16,9 @@ name: Build e2e tests
 
 on:
   push:
-    branches: [ develop, release/* ]
+    branches: [ develop, release/** ]
   pull_request:
-      branches: [ develop, release/* ]
+      branches: [ develop, release/** ]
       types: [ opened, synchronize, reopened, labeled ]
   workflow_dispatch:
 
@@ -45,7 +45,7 @@ jobs:
 
     steps:
       # Pinned 1.0.0 version
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: plugin
           submodules: 'recursive'
@@ -61,14 +61,14 @@ jobs:
               - '${{ matrix.module }}/**/e2e-test/**'
 
       - name: Checkout e2e test repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: cdapio/cdap-e2e-tests
           path: e2e
           ref: release/6.10
 
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
@@ -96,6 +96,12 @@ jobs:
             POSTGRESQL_USERNAME:cdapio-github-builds/POSTGRESQL_USERNAME
             POSTGRESQL_PASSWORD:cdapio-github-builds/POSTGRESQL_PASSWORD
             POSTGRESQL_PORT:cdapio-github-builds/POSTGRESQL_PORT
+            CLOUDSQL_POSTGRESQL_USERNAME:cdapio-github-builds/CLOUDSQL_POSTGRESQL_USERNAME
+            CLOUDSQL_POSTGRESQL_PASSWORD:cdapio-github-builds/CLOUDSQL_POSTGRESQL_PASSWORD
+            CLOUDSQL_POSTGRESQL_CONNECTION_NAME:cdapio-github-builds/CLOUDSQL_POSTGRESQL_CONNECTION_NAME
+            CLOUDSQL_MYSQL_USERNAME:cdapio-github-builds/CLOUDSQL_MYSQL_USERNAME
+            CLOUDSQL_MYSQL_PASSWORD:cdapio-github-builds/CLOUDSQL_MYSQL_PASSWORD
+            CLOUDSQL_MYSQL_CONNECTION_NAME:cdapio-github-builds/CLOUDSQL_MYSQL_CONNECTION_NAME
 
       - name: Run required e2e tests
         if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false'
@@ -117,6 +123,12 @@ jobs:
           POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.POSTGRESQL_USERNAME }}
           POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.POSTGRESQL_PASSWORD }}
           POSTGRESQL_PORT: ${{ steps.secrets.outputs.POSTGRESQL_PORT }}
+          CLOUDSQL_POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_USERNAME }}
+          CLOUDSQL_POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_PASSWORD }}
+          CLOUDSQL_POSTGRESQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_CONNECTION_NAME }}
+          CLOUDSQL_MYSQL_USERNAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_USERNAME }}
+          CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }}
+          CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }}
 
       - name: Run all e2e tests
         if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true'
@@ -138,25 +150,28 @@ jobs:
           POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.POSTGRESQL_USERNAME }}
           POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.POSTGRESQL_PASSWORD }}
           POSTGRESQL_PORT: ${{ steps.secrets.outputs.POSTGRESQL_PORT }}
-
-      - name: Upload report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: Cucumber report - ${{ matrix.module }}
-          path: ./**/target/cucumber-reports
+          CLOUDSQL_POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_USERNAME }}
+          CLOUDSQL_POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_PASSWORD }}
+          CLOUDSQL_POSTGRESQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_POSTGRESQL_CONNECTION_NAME }}
+          CLOUDSQL_MYSQL_USERNAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_USERNAME }}
+          CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }}
+          CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }}
 
       - name: Upload debug files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: Debug files - ${{ matrix.module }}
           path: ./**/target/e2e-debug
 
       - name: Upload files to GCS
-        uses: google-github-actions/upload-cloud-storage@v0
+        uses: google-github-actions/upload-cloud-storage@v2
         if: always()
         with:
           path: ./plugin
           destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}
           glob: '**/target/cucumber-reports/**'
+
+      - name: Cucumber Report URL
+        if: always()
+        run: echo "https://storage.googleapis.com/e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}/plugin/${{ matrix.module }}/target/cucumber-reports/advanced-reports/cucumber-html-reports/overview-features.html"

database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java

@@ -45,6 +45,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC
   public static final String CONNECTION_ARGUMENTS = "connectionArguments";
   public static final String JDBC_PLUGIN_NAME = "jdbcPluginName";
   public static final String JDBC_PLUGIN_TYPE = "jdbc";
+  public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel";
 
   @Name(JDBC_PLUGIN_NAME)
   @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " +

database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java

@@ -20,8 +20,9 @@
 import io.cdap.cdap.api.annotation.Macro;
 import io.cdap.cdap.api.annotation.Name;
 import io.cdap.plugin.db.ConnectionConfig;
+import io.cdap.plugin.db.TransactionIsolationLevel;
 
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
 
@@ -42,6 +43,12 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec
   @Nullable
   protected Integer port;
 
+  @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL)
+  @Description("The transaction isolation level for the database session.")
+  @Macro
+  @Nullable
+  protected String transactionIsolationLevel;
+
   public String getHost() {
     return host;
   }
@@ -55,4 +62,21 @@ public int getPort() {
   public boolean canConnect() {
     return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT);
   }
+
+  @Override
+  public Map<String, String> getAdditionalArguments() {
+    Map<String, String> additonalArguments = new HashMap<>();
+    if (getTransactionIsolationLevel() != null) {
+      additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel());
+    }
+    return additonalArguments;
+  }
+
+  public String getTransactionIsolationLevel() {
+    if (transactionIsolationLevel == null) {
+      return null;
+    }
+    return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name();
+  }
 }
+

mssql-plugin/docs/SQL Server-connector.md

@@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication.
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the database connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16)
+
 **Authentication Type:** Indicates which authentication method will be used for the connection. Use 'SQL Login'. to
 connect to a SQL Server using username and password properties. Use 'Active Directory Password' to connect to an Azure
 SQL Database/Data Warehouse using an Azure AD principal name and password.

mssql-plugin/docs/SqlServer-batchsink.md

@@ -46,6 +46,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the database connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16)
+
 **Instance Name:** SQL Server instance name to connect to. When it is not specified, a
 connection is made to the default instance. For the case where both the instanceName and port are specified,
 see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot

mssql-plugin/docs/SqlServer-batchsource.md

@@ -56,6 +56,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the database connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16)
+
 **Instance Name:** SQL Server instance name to connect to. When it is not specified, a
 connection is made to the default instance. For the case where both the instanceName and port are specified,
 see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot

mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java

@@ -167,6 +167,11 @@ public Map<String, String> getDBSpecificArguments() {
                                                          packetSize, queryTimeout);
     }
 
+    @Override
+    public String getTransactionIsolationLevel() {
+      return connection.getTransactionIsolationLevel();
+    }
+
     @Override
     public String getConnectionString() {
       return String.format(SqlServerConstants.SQL_SERVER_CONNECTION_STRING_FORMAT,

mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java

@@ -188,6 +188,11 @@ public List<String> getInitQueries() {
       return Collections.emptyList();
     }
 
+    @Override
+    public String getTransactionIsolationLevel() {
+      return connection.getTransactionIsolationLevel();
+    }
+
     @Override
     public void validate(FailureCollector collector) {
       ConfigUtil.validateConnection(this, useConnection, connection, collector);

mssql-plugin/widgets/SQL Server-connector.json

@@ -64,6 +64,20 @@
           "widget-type": "password",
           "label": "Password",
           "name": "password"
+        },
+        {
+          "widget-type": "select",
+          "label": "Transaction Isolation Level",
+          "name": "transactionIsolationLevel",
+          "widget-attributes": {
+            "values": [
+              "TRANSACTION_READ_UNCOMMITTED",
+              "TRANSACTION_READ_COMMITTED",
+              "TRANSACTION_REPEATABLE_READ",
+              "TRANSACTION_SERIALIZABLE"
+            ],
+            "default": "TRANSACTION_SERIALIZABLE"
+          }
         }
       ]
     },

mssql-plugin/widgets/SqlServer-batchsink.json

@@ -84,6 +84,20 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "select",
+          "label": "Transaction Isolation Level",
+          "name": "transactionIsolationLevel",
+          "widget-attributes": {
+            "values": [
+              "TRANSACTION_READ_UNCOMMITTED",
+              "TRANSACTION_READ_COMMITTED",
+              "TRANSACTION_REPEATABLE_READ",
+              "TRANSACTION_SERIALIZABLE"
+            ],
+            "default": "TRANSACTION_SERIALIZABLE"
+          }
+        },
         {
           "widget-type": "keyvalue",
           "label": "Connection Arguments",
@@ -267,6 +281,10 @@
         {
           "type": "property",
           "name": "connectionArguments"
+        },
+        {
+          "type": "property",
+          "name": "transactionIsolationLevel"
         }
       ]
     },

mssql-plugin/widgets/SqlServer-batchsource.json

@@ -84,6 +84,20 @@
           "label": "Password",
           "name": "password"
         },
+        {
+          "widget-type": "select",
+          "label": "Transaction Isolation Level",
+          "name": "transactionIsolationLevel",
+          "widget-attributes": {
+            "values": [
+              "TRANSACTION_READ_UNCOMMITTED",
+              "TRANSACTION_READ_COMMITTED",
+              "TRANSACTION_REPEATABLE_READ",
+              "TRANSACTION_SERIALIZABLE"
+            ],
+            "default": "TRANSACTION_SERIALIZABLE"
+          }
+        },
         {
           "widget-type": "keyvalue",
           "label": "Connection Arguments",
@@ -316,6 +330,10 @@
         {
           "type": "property",
           "name": "connectionArguments"
+        },
+        {
+          "type": "property",
+          "name": "transactionIsolationLevel"
         }
       ]
     },

mysql-plugin/docs/MySQL-connector.md

@@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication.
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the databse connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in MySQL, refer to the  [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html)
+
 **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments
 will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations.
 This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies

mysql-plugin/docs/Mysql-batchsink.md

@@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}.
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the databse connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in MySQL, refer to the  [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html)
+
 **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments
 will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations.
 

mysql-plugin/docs/Mysql-batchsource.md

@@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s
 
 **Password:** Password to use to connect to the specified database.
 
+**Transaction Isolation Level** The transaction isolation level of the database connection
+- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible.
+- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented.
+- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible.
+- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible.
+
+For more details on the Transaction Isolation Levels supported in MySQL, refer to the  [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html)
+
 **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments
 will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations.