Skip to content

Commit fd593c4

Browse files
yjhawaryjhawar
committed
Added e2e tests with snapshot and cdc
1 parent 6994f86 commit fd593c4

15 files changed

+1008
-0
lines changed

‎pom.xml

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,129 @@
385385
</plugins>
386386
</build>
387387
</profile>
388+
<profile>
389+
<id>e2e-tests</id>
390+
<properties>
391+
<testSourceLocation>src/e2e-test/java</testSourceLocation>
392+
<TEST_RUNNER>TestRunner.java</TEST_RUNNER>
393+
</properties>
394+
<build>
395+
<testResources>
396+
<testResource>
397+
<directory>src/e2e-test/resources</directory>
398+
</testResource>
399+
</testResources>
400+
<plugins>
401+
<plugin>
402+
<groupId>org.apache.maven.plugins</groupId>
403+
<artifactId>maven-surefire-plugin</artifactId>
404+
<version>2.18.1</version>
405+
<configuration>
406+
<skipTests>true</skipTests>
407+
</configuration>
408+
</plugin>
409+
410+
<plugin>
411+
<groupId>org.apache.maven.plugins</groupId>
412+
<artifactId>maven-failsafe-plugin</artifactId>
413+
<version>3.0.0-M5</version>
414+
<configuration>
415+
<includes>
416+
<include>${TEST_RUNNER}</include>
417+
</includes>
418+
<!--Start configuration to run TestRunners in parallel-->
419+
<parallel>classes</parallel> <!--Running TestRunner classes in parallel-->
420+
<threadCount>2</threadCount> <!--Number of classes to run in parallel-->
421+
<forkCount>2</forkCount> <!--Number of JVM processes -->
422+
<reuseForks>true</reuseForks>
423+
<!--End configuration to run TestRunners in parallel-->
424+
<environmentVariables>
425+
<GOOGLE_APPLICATION_CREDENTIALS>
426+
${GOOGLE_APPLICATION_CREDENTIALS}
427+
</GOOGLE_APPLICATION_CREDENTIALS>
428+
<SERVICE_ACCOUNT_TYPE>
429+
${SERVICE_ACCOUNT_TYPE}
430+
</SERVICE_ACCOUNT_TYPE>
431+
<SERVICE_ACCOUNT_FILE_PATH>
432+
${SERVICE_ACCOUNT_FILE_PATH}
433+
</SERVICE_ACCOUNT_FILE_PATH>
434+
<SERVICE_ACCOUNT_JSON>
435+
${SERVICE_ACCOUNT_JSON}
436+
</SERVICE_ACCOUNT_JSON>
437+
</environmentVariables>
438+
</configuration>
439+
<executions>
440+
<execution>
441+
<goals>
442+
<goal>integration-test</goal>
443+
</goals>
444+
</execution>
445+
</executions>
446+
</plugin>
447+
448+
<plugin>
449+
<groupId>net.masterthought</groupId>
450+
<artifactId>maven-cucumber-reporting</artifactId>
451+
<version>5.5.0</version>
452+
453+
<executions>
454+
<execution>
455+
<id>execution</id>
456+
<phase>verify</phase>
457+
<goals>
458+
<goal>generate</goal>
459+
</goals>
460+
<configuration>
461+
<projectName>Cucumber Reports</projectName> <!-- Replace with project name -->
462+
<outputDirectory>target/cucumber-reports/advanced-reports</outputDirectory>
463+
<buildNumber>1</buildNumber>
464+
<skip>false</skip>
465+
<inputDirectory>${project.build.directory}/cucumber-reports</inputDirectory>
466+
<jsonFiles> <!-- supports wildcard or name pattern -->
467+
<param>**/*.json</param>
468+
</jsonFiles> <!-- optional, defaults to outputDirectory if not specified -->
469+
<classificationDirectory>${project.build.directory}/cucumber-reports</classificationDirectory>
470+
<checkBuildResult>true</checkBuildResult>
471+
</configuration>
472+
</execution>
473+
</executions>
474+
</plugin>
475+
</plugins>
476+
</build>
477+
478+
<dependencyManagement>
479+
<dependencies>
480+
<dependency>
481+
<groupId>com.google.guava</groupId>
482+
<artifactId>guava</artifactId>
483+
<version>31.1-jre</version>
484+
</dependency>
485+
</dependencies>
486+
</dependencyManagement>
487+
488+
<dependencies>
489+
<dependency>
490+
<groupId>com.oracle.database.jdbc</groupId>
491+
<artifactId>ojdbc8</artifactId>
492+
<version>21.1.0.0</version>
493+
<scope>test</scope>
494+
</dependency>
495+
<dependency>
496+
<groupId>io.cdap.tests.e2e</groupId>
497+
<artifactId>cdap-e2e-framework</artifactId>
498+
<version>0.3.0-SNAPSHOT</version>
499+
<scope>test</scope>
500+
</dependency>
501+
<dependency>
502+
<groupId>ch.qos.logback</groupId>
503+
<artifactId>logback-classic</artifactId>
504+
<version>1.2.8</version>
505+
<scope>runtime</scope>
506+
</dependency>
507+
</dependencies>
508+
509+
</profile>
510+
388511
</profiles>
389512

390513
</project>

‎src/e2e-test/features/Pipeline.feature

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
Feature: Oracle - Verify Oracle source data transfer
18+
19+
# Scenario: Sanity test from Oracle to Big Query
20+
# Given Open DataFusion Project with replication to configure pipeline
21+
# When Enter input plugin property: "name" with value: "pipelineName"
22+
# And Click on the Next button
23+
# And Select Oracle as Source
24+
# Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
25+
# Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
26+
# Then Click plugin property: "region"
27+
# Then Click plugin property: "regionOption"
28+
# Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
29+
# Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
30+
# Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields
31+
# Then Click on the Next button
32+
# Then Replace input plugin property: "loadInterval" with value: "loadInterval"
33+
# Then Click on the Next button
34+
# Then Validate Table is available and can be selected "ABC.E2E-sanity"
35+
# And Click on the Next button
36+
# And Click on the Next button
37+
# And Click on the Next button
38+
# Then Deploy the replication pipeline
39+
# And Run the replication Pipeline
40+
# Then Open the logs
41+
# And Wait till pipeline is in running state and check if no errors occurred
42+
# Then Verify expected Oracle records in target BigQuery table
43+
# And Capture raw logs
44+
# Then Close the pipeline logs and stop the pipeline
45+
46+
# @ORACLE_SOURCE
47+
Scenario: To verify snapshot and cdc from Oracle to Big Query successfully
48+
Given Open DataFusion Project with replication to configure pipeline
49+
When Enter input plugin property: "name" with value: "pipelineName"
50+
And Click on the Next button
51+
And Select Oracle as Source
52+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
53+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
54+
Then Click plugin property: "region"
55+
Then Click plugin property: "regionOption"
56+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
57+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
58+
Then Replace input plugin property: "sid" with value: "database" for Credentials and Authorization related fields
59+
Then Click on the Next button
60+
Then Replace input plugin property: "loadInterval" with value: "loadInterval"
61+
Then Click on the Next button
62+
Then Validate Table is available and can be selected "ABC.E2E1"
63+
And Click on the Next button
64+
And Click on the Next button
65+
And Click on the Next button
66+
Then Deploy the replication pipeline
67+
And Run the replication Pipeline
68+
Then Open the logs
69+
And Wait till pipeline is in running state and check if no errors occurred
70+
Then Verify expected Oracle records in target BigQuery table
71+
And Insert a record in the source table and wait for replication
72+
Then Verify expected Oracle records in target BigQuery table
73+
And Delete a record in the source table and wait for replication
74+
Then Verify expected Oracle records in target BigQuery table
75+
And Update a record in the source table and wait for replication
76+
Then Verify expected Oracle records in target BigQuery table
77+
And Capture raw logs
78+
Then Close the pipeline logs and stop the pipeline

‎src/e2e-test/java/io.cdap.plugin/actions/ReplicationActions.java

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
* Copyright (c) 2023.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.actions;
18+
19+
import io.cdap.e2e.pages.actions.CdfPipelineRunAction;
20+
import io.cdap.e2e.pages.locators.CdfPipelineRunLocators;
21+
import io.cdap.e2e.utils.*;
22+
import io.cdap.plugin.locators.ReplicationLocators;
23+
import io.cdap.plugin.utils.OracleClient;
24+
import io.cdap.plugin.utils.ValidationHelper;
25+
import org.apache.commons.lang.StringUtils;
26+
import org.junit.Assert;
27+
import stepsdesign.BeforeActions;
28+
29+
import java.io.IOException;
30+
import java.sql.SQLException;
31+
import java.util.ArrayList;
32+
import java.util.List;
33+
import java.util.Map;
34+
import java.util.concurrent.TimeUnit;
35+
36+
public class ReplicationActions {
37+
private static String parentWindow = StringUtils.EMPTY;
38+
private static final String projectId = PluginPropertyUtils.pluginProp("projectId");
39+
private static final String database = PluginPropertyUtils.pluginProp("database");
40+
public static String tableName = PluginPropertyUtils.pluginProp("sourceTable");
41+
public static String schemaName = PluginPropertyUtils.pluginProp("schema");
42+
public static String datatypeColumnNames = PluginPropertyUtils.pluginProp("datatypeColumnNames");
43+
public static String datatypeValues = PluginPropertyUtils.pluginProp("datatypeValuesForInsertOperation");
44+
public static String deleteCondition = PluginPropertyUtils.pluginProp("deleteRowCondition");
45+
public static String updateCondition = PluginPropertyUtils.pluginProp("updateRowCondition");
46+
public static String updatedValue = PluginPropertyUtils.pluginProp("updatedRow");
47+
48+
static {
49+
SeleniumHelper.getPropertiesLocators(ReplicationLocators.class);
50+
}
51+
public static void clickNextButton() throws InterruptedException {
52+
TimeUnit time = TimeUnit.SECONDS;
53+
time.sleep(1);
54+
ElementHelper.clickOnElement(ReplicationLocators.next);
55+
}
56+
public static void clickOnOraclePlugin() {
57+
ElementHelper.clickOnElement(ReplicationLocators.oraclePlugin);
58+
}
59+
60+
public static void selectTable(String tableName) {
61+
WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.selectTable(tableName));
62+
AssertionHelper.verifyElementDisplayed(ReplicationLocators.selectTable(tableName));
63+
ElementHelper.clickOnElement(ReplicationLocators.selectTable(tableName));
64+
}
65+
66+
public static void deployPipeline() {
67+
ElementHelper.clickOnElement(ReplicationLocators.deployPipeline);
68+
}
69+
70+
public static void startPipeline() {
71+
ElementHelper.clickIfDisplayed(ReplicationLocators.start, ConstantsUtil.DEFAULT_TIMEOUT_SECONDS);
72+
}
73+
74+
public static void runThePipeline() {
75+
startPipeline();
76+
WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.running);
77+
}
78+
79+
public static void openAdvanceLogs() {
80+
ReplicationLocators.logs.click();
81+
parentWindow = SeleniumDriver.getDriver().getWindowHandle();
82+
ArrayList<String> tabs = new ArrayList(SeleniumDriver.getDriver().getWindowHandles());
83+
SeleniumDriver.getDriver().switchTo().window(tabs.get(tabs.indexOf(parentWindow) + 1));
84+
ReplicationLocators.advancedLogs.click();
85+
}
86+
87+
public static void captureRawLog() {
88+
//Capturing raw logs.
89+
try {
90+
String rawLogs = getRawLogs();
91+
String logsSeparatorMessage = ConstantsUtil.LOGS_SEPARATOR_MESSAGE
92+
.replace("MESSAGE", "DEPLOYED PIPELINE RUNTIME LOGS");
93+
BeforeActions.scenario.write(rawLogs);
94+
CdfPipelineRunAction.writeRawLogsToFile(BeforeActions.file, logsSeparatorMessage, rawLogs);
95+
} catch (Exception e) {
96+
BeforeActions.scenario.write("Exception in capturing logs : " + e);
97+
}
98+
}
99+
100+
public static String getRawLogs() {
101+
CdfPipelineRunAction.viewRawLogs();
102+
ArrayList<String> tabs = new ArrayList(SeleniumDriver.getDriver().getWindowHandles());
103+
PageHelper.switchToWindow(tabs.indexOf(parentWindow) + 2);
104+
String logs = CdfPipelineRunLocators.logsTextbox.getText();
105+
Assert.assertNotNull(logs);
106+
PageHelper.closeCurrentWindow();
107+
return logs;
108+
}
109+
110+
public static void waitTillPipelineIsRunningAndCheckForErrors() throws InterruptedException {
111+
//wait for datastream to startup
112+
int defaultTimeout = Integer.parseInt(PluginPropertyUtils.pluginProp("datastream.timeout"));
113+
TimeUnit time = TimeUnit.SECONDS;
114+
time.sleep(defaultTimeout);
115+
// Checking if an error message is displayed.
116+
Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error));
117+
}
118+
119+
public static void closeTheLogsAndClickOnStopButton() {
120+
//As the logs get opened in a new window in this plugin so after closing them we have to switch to parent window.
121+
SeleniumDriver.getDriver().switchTo().window(parentWindow);
122+
//Stopping the pipeline
123+
ElementHelper.clickOnElement(ReplicationLocators.stop);
124+
WaitHelper.waitForElementToBeDisplayed(ReplicationLocators.stopped);
125+
}
126+
public static void verifyTargetBigQueryRecordMatchesExpectedOracleRecord()
127+
throws IOException, InterruptedException, SQLException, ClassNotFoundException {
128+
// Checking if an error message is displayed.
129+
Assert.assertFalse(ElementHelper.isElementDisplayed(ReplicationLocators.error));
130+
131+
List<Map<String, Object>> sourceOracleRecords = OracleClient.getOracleRecordsAsMap(tableName, schemaName);
132+
List<Map<String, Object>> targetBigQueryRecords = ValidationHelper.getBigQueryRecordsAsMap(projectId, database, tableName); //+ "_v1`"
133+
ValidationHelper.validateRecords(sourceOracleRecords, targetBigQueryRecords);
134+
}
135+
136+
public static void insertRecordAndWait()
137+
throws IOException, InterruptedException, SQLException, ClassNotFoundException { //JCoException,
138+
OracleClient.insertRow(tableName,schemaName, datatypeColumnNames,datatypeValues);
139+
OracleClient.forceFlushCDC();
140+
ValidationHelper.waitForFlush();
141+
}
142+
143+
public static void deleteRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException {
144+
OracleClient.deleteRow(tableName,schemaName, deleteCondition);
145+
OracleClient.forceFlushCDC();
146+
ValidationHelper.waitForFlush();
147+
}
148+
149+
public static void updateRecordAndWait() throws SQLException, ClassNotFoundException, IOException, InterruptedException {
150+
OracleClient.updateRow(tableName,schemaName, updateCondition, updatedValue );
151+
OracleClient.forceFlushCDC();
152+
ValidationHelper.waitForFlush();
153+
}
154+
}

‎src/e2e-test/java/io.cdap.plugin/actions/package-info.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Copyright (c) 2023.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.actions;

0 commit comments

Comments
 (0)