diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ce0eb526..686ff7b00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,8 +10,8 @@ # the License. # This workflow will build a Java project with Maven -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven -# Note: Any changes to this workflow would be used only after merging into develop +# Fors more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# Note: Any changes to this workflow would be used only after merging into developss name: Build with unit tests on: @@ -23,7 +23,7 @@ on: jobs: build: - runs-on: k8s-runner-build + runs-on: self-hosted if: ${{ github.event.workflow_run.conclusion != 'skipped' }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a49e68043..07a40d059 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,7 +9,7 @@ # License for the specific language governing permissions and limitations under # the License. -# This workflow will build a Java project with Maven +# This workflow will build a Java project with Mavens # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven # Note: Any changes to this workflow would be used only after merging into develop name: Build e2e tests @@ -24,7 +24,7 @@ on: jobs: build: - runs-on: k8s-runner-e2e + runs-on: self-hosted # We allow builds: # 1) When triggered manually # 2) When it's a merge into a branch @@ -63,8 +63,9 @@ jobs: - name: Checkout e2e test repo uses: actions/checkout@v3 with: - repository: cdapio/cdap-e2e-tests + repository: cloudsufi/cdap-e2e-tests path: e2e + ref: oracle_debug - name: Cache uses: actions/cache@v3 @@ -79,28 +80,28 @@ jobs: uses: 'google-github-actions/get-secretmanager-secrets@v0' with: secrets: |- - ORACLE_HOST:cdapio-github-builds/ORACLE_HOST - ORACLE_USERNAME:cdapio-github-builds/ORACLE_USERNAME - ORACLE_PASSWORD:cdapio-github-builds/ORACLE_PASSWORD - ORACLE_PORT:cdapio-github-builds/ORACLE_PORT - MSSQL_HOST:cdapio-github-builds/MSSQL_HOST - MSSQL_USERNAME:cdapio-github-builds/MSSQL_USERNAME - MSSQL_PASSWORD:cdapio-github-builds/MSSQL_PASSWORD - MSSQL_PORT:cdapio-github-builds/MSSQL_PORT - MYSQL_HOST:cdapio-github-builds/MYSQL_HOST - MYSQL_USERNAME:cdapio-github-builds/MYSQL_USERNAME - MYSQL_PASSWORD:cdapio-github-builds/MYSQL_PASSWORD - MYSQL_PORT:cdapio-github-builds/MYSQL_PORT - POSTGRESQL_HOST:cdapio-github-builds/POSTGRESQL_HOST - POSTGRESQL_USERNAME:cdapio-github-builds/POSTGRESQL_USERNAME - POSTGRESQL_PASSWORD:cdapio-github-builds/POSTGRESQL_PASSWORD - POSTGRESQL_PORT:cdapio-github-builds/POSTGRESQL_PORT - CLOUDSQL_POSTGRESQL_USERNAME:cdapio-github-builds/CLOUDSQL_POSTGRESQL_USERNAME - CLOUDSQL_POSTGRESQL_PASSWORD:cdapio-github-builds/CLOUDSQL_POSTGRESQL_PASSWORD - CLOUDSQL_POSTGRESQL_CONNECTION_NAME:cdapio-github-builds/CLOUDSQL_POSTGRESQL_CONNECTION_NAME - CLOUDSQL_MYSQL_USERNAME:cdapio-github-builds/CLOUDSQL_MYSQL_USERNAME - CLOUDSQL_MYSQL_PASSWORD:cdapio-github-builds/CLOUDSQL_MYSQL_PASSWORD - CLOUDSQL_MYSQL_CONNECTION_NAME:cdapio-github-builds/CLOUDSQL_MYSQL_CONNECTION_NAME + ORACLE_HOST:cdf-entcon/ORACLE_HOST + ORACLE_USERNAME:cdf-entcon/ORACLE_USERNAME + ORACLE_PASSWORD:cdf-entcon/ORACLE_PASSWORD + ORACLE_PORT:cdf-entcon/ORACLE_PORT + MSSQL_HOST:cdf-entcon/MSSQL_HOST + MSSQL_USERNAME:cdf-entcon/MSSQL_USERNAME + MSSQL_PASSWORD:cdf-entcon/MSSQL_PASSWORD + MSSQL_PORT:cdf-entcon/MSSQL_PORT + MYSQL_HOST:cdf-entcon/MYSQL_HOST + MYSQL_USERNAME:cdf-entcon/MYSQL_USERNAME + MYSQL_PASSWORD:cdf-entcon/MYSQL_PASSWORD + MYSQL_PORT:cdf-entcon/MYSQL_PORT + POSTGRESQL_HOST:cdf-entcon/POSTGRESQL_HOST + POSTGRESQL_USERNAME:cdf-entcon/POSTGRESQL_USERNAME + POSTGRESQL_PASSWORD:cdf-entcon/POSTGRESQL_PASSWORD + POSTGRESQL_PORT:cdf-entcon/POSTGRESQL_PORT + CLOUDSQL_POSTGRESQL_USERNAME:cdf-entcon/CLOUDSQL_POSTGRESQL_USERNAME + CLOUDSQL_POSTGRESQL_PASSWORD:cdf-entcon/CLOUDSQL_POSTGRESQL_PASSWORD + CLOUDSQL_POSTGRESQL_CONNECTION_NAME:cdf-entcon/CLOUDSQL_POSTGRESQL_CONNECTION_NAME + CLOUDSQL_MYSQL_USERNAME:cdf-entcon/CLOUDSQL_MYSQL_USERNAME + CLOUDSQL_MYSQL_PASSWORD:cdf-entcon/CLOUDSQL_MYSQL_PASSWORD + CLOUDSQL_MYSQL_CONNECTION_NAME:cdf-entcon/CLOUDSQL_MYSQL_CONNECTION_NAME - name: Run required e2e tests if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false' diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml index 11db8ac25..a55d472d8 100644 --- a/.github/workflows/trigger.yml +++ b/.github/workflows/trigger.yml @@ -26,7 +26,7 @@ on: jobs: trigger: - runs-on: ubuntu-latest + runs-on: self-hosted # We allow builds: # 1) When triggered manually diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties index 9a6181214..34c6b5023 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -17,7 +17,7 @@ fetchSize=1000 NumSplits=1 SplitBy=ID jdbcURL=jdbc:mysql:///%s?cloudSqlInstance=%s&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=%s&password=%s -projectId=cdf-athena +projectId=cdf-entcon datasetprojectId=cdf-athena BQReferenceName=reference targetTable=mytable5 diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/pluginParameters.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/pluginParameters.properties index 3212af0a9..b1b6514e7 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -79,7 +79,7 @@ splitByColumn=ID importQuery=where $CONDITIONS #bq properties -projectId=cdf-athena +projectId=cdf-entcon dataset=test_automation bqOutputMultipleDatatypesSchema=[{"key":"col1","value":"bytes"},{"key":"col2","value":"string"},\ {"key":"col3","value":"date"},{"key":"col4","value":"double"},{"key":"col5","value":"decimal"},\ diff --git a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties index ede609c0e..177b7c7f3 100644 --- a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -22,8 +22,8 @@ zeroValue=0 blankvalue= invalidTableName=Table123@ bqtarget.table=target-table -projectId=cdf-athena -datasetprojectId=cdf-athena +projectId=cdf-entcon +datasetprojectId=cdf-entcon dataset=SQL_SERVER_TEST invalid.boundQuery=SELECT MIN(id),MAX(id) FROM table splitby=id diff --git a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties index 215e4b830..912dc7fcc 100644 --- a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -14,12 +14,12 @@ invalid.tablename=123# invalid.referenceName=abc# fetchsize=1000 no.of.splits=1 -projectId=cdf-athena +projectId=cdf-entcon #invalid properties invalid.host=localhost1 bqTargetTable=target-table bqSourceTable=dummy -datasetprojectId=cdf-athena +datasetprojectId=cdf-entcon dataset=test_automation table.name=mysqltable importQuery=select * from testTable where $CONDITIONS; diff --git a/oracle-plugin/src/e2e-test/features/oracle/Datatype.feature b/oracle-plugin/src/e2e-test/features/oracle/Datatype.feature index d19e77d85..aa925ae79 100644 --- a/oracle-plugin/src/e2e-test/features/oracle/Datatype.feature +++ b/oracle-plugin/src/e2e-test/features/oracle/Datatype.feature @@ -17,7 +17,7 @@ @Oracle Feature: Oracle - Verify Oracle source data transfer for multiple datatypes @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required - # Oracle Sanity test to transfer table data containing multiple datatypes + # Oracle Sanitys test to transfer table data containing multiple datatypes Scenario: To verify data is getting transferred from Oracle to Oracle successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/oracle-plugin/src/e2e-test/features/oracle/DatatypeLong.feature b/oracle-plugin/src/e2e-test/features/oracle/DatatypeLong.feature index 1eed6713f..0ab0b28bc 100644 --- a/oracle-plugin/src/e2e-test/features/oracle/DatatypeLong.feature +++ b/oracle-plugin/src/e2e-test/features/oracle/DatatypeLong.feature @@ -17,7 +17,7 @@ @Oracle Feature: Oracle - Verify Oracle source data transfer of type LONG @ORACLE_SOURCE_DATATYPES_TEST2 @ORACLE_TARGET_DATATYPES_TEST2 @Oracle_Required - # Oracle Sanity test to transfer table data containing LONG + # Oracles Sanity test to transfer table data containing LONG Scenario: To verify data is getting transferred from Oracle to Oracle successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/oracle-plugin/src/e2e-test/features/oracle/DatatypeLongVarchar.feature b/oracle-plugin/src/e2e-test/features/oracle/DatatypeLongVarchar.feature index dd75e445b..e7144378c 100644 --- a/oracle-plugin/src/e2e-test/features/oracle/DatatypeLongVarchar.feature +++ b/oracle-plugin/src/e2e-test/features/oracle/DatatypeLongVarchar.feature @@ -17,7 +17,7 @@ @Oracle Feature: Oracle - Verify Oracle source data transfer of type LONG VARCHAR @ORACLE_SOURCE_DATATYPES_TEST4 @ORACLE_TARGET_DATATYPES_TEST4 @Oracle_Required - # Oracle Sanity test to transfer table data containing LONG VARCHAR + # Oracles Sanitysss test to transfer table data containing LONG VARCHAR Scenario: To verify data is getting transferred from Oracle to Oracle successfully Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" diff --git a/oracle-plugin/src/e2e-test/features/oracle/Oracle.feature b/oracle-plugin/src/e2e-test/features/oracle/Oracle.feature index 3aecd62b9..574d741fd 100644 --- a/oracle-plugin/src/e2e-test/features/oracle/Oracle.feature +++ b/oracle-plugin/src/e2e-test/features/oracle/Oracle.feature @@ -7,7 +7,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless required by applicable law or agreed to in writing, softwares # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under diff --git a/oracle-plugin/src/e2e-test/features/source/OracleDesignTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleDesignTime.feature index 5bd52b7a5..70f81ed6f 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleDesignTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleDesignTime.feature @@ -15,7 +15,7 @@ # @Oracle @Oracle_Required -Feature: Oracle source- Verify Oracle source plugin design time scenarios +Feature: Oracle source- Verify Oracle source plugin design time scenarioss @ORACLE_SOURCE_TEST Scenario: To verify Oracle source plugin validation with connection and basic details for connectivity diff --git a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties index 88dc3fe34..1d263d51f 100644 --- a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties @@ -1,8 +1,8 @@ driverName=oracle -databaseName=ORCLPDB1 +databaseName=ORCLCDB sourceRef=source targetRef=target -schema=CDAP +schema=c##builduser host=ORACLE_HOST port=ORACLE_PORT username=ORACLE_USERNAME @@ -95,7 +95,7 @@ splitByColumn=ID importQuery=where $CONDITIONS #bq properties -projectId=cdf-athena +projectId=cdf-entcon dataset=test_automation bqOutputDatatypesSchema=[{"key":"ID","value":"decimal"},{"key":"LASTNAME","value":"string"}] jdbcUrl=jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=%s;OAuthType=3; diff --git a/postgresql-plugin/src/e2e-test/resources/pluginParameters.properties b/postgresql-plugin/src/e2e-test/resources/pluginParameters.properties index cf132a9e5..40a6ac8d6 100644 --- a/postgresql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/postgresql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -79,7 +79,7 @@ zeroValue=0 splitByColumn=ID importQuery=where $CONDITIONS #bq properties -projectId=cdf-athena +projectId=cdf-entcon dataset=test_automation bqOutputMultipleDatatypesSchema=[{"key":"col1","value":"bytes"},{"key":"col2","value":"string"},\ {"key":"col3","value":"date"},{"key":"col4","value":"double"},{"key":"col5","value":"decimal"},\