Skip to content

Commit 9b2e1d2

Browse files
itsmekumariitsmekumari
authored
e2e_mysql replication tests (#245)
1 parent 606acef commit 9b2e1d2

File tree

19 files changed

+1221
-0
lines changed

19 files changed

+1221
-0
lines changed

‎.github/workflows/e2e.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Copyright © 2023 Cask Data, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
3+
# use this file except in compliance with the License. You may obtain a copy of
4+
# the License at
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software
7+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
8+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
9+
# License for the specific language governing permissions and limitations under
10+
# the License.
11+
12+
# This workflow will build a Java project with Maven
13+
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
14+
# Note: Any changes to this workflow would be used only after merging into develop
15+
name: Build e2e tests
16+
17+
on:
18+
push:
19+
branches: [ develop ]
20+
pull_request:
21+
branches: [ develop ]
22+
types: [ opened, synchronize, reopened, labeled ]
23+
workflow_dispatch:
24+
25+
jobs:
26+
build:
27+
runs-on: k8s-runner-e2e
28+
# We allow builds:
29+
# 1) When triggered manually
30+
# 2) When it's a merge into a branch
31+
# 3) For PRs that are labeled as build and
32+
# - It's a code change
33+
# - A build label was just added
34+
# A bit complex, but prevents builds when other labels are manipulated
35+
if: >
36+
github.event_name == 'workflow_dispatch'
37+
|| github.event_name == 'push'
38+
|| (contains(github.event.pull_request.labels.*.name, 'build')
39+
&& (github.event.action != 'labeled' || github.event.label.name == 'build')
40+
)
41+
strategy:
42+
matrix:
43+
module: [mysql-delta-plugins]
44+
fail-fast: false
45+
46+
steps:
47+
# Pinned 1.0.0 version
48+
- uses: actions/checkout@v3
49+
with:
50+
path: plugin
51+
submodules: 'recursive'
52+
ref: ${{ github.event.workflow_run.head_sha }}
53+
54+
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
55+
if: github.event_name != 'workflow_dispatch' && github.event_name != 'push'
56+
id: filter
57+
with:
58+
working-directory: plugin
59+
filters: |
60+
e2e-test:
61+
- '${{ matrix.module }}/**/e2e-test/**'
62+
63+
- name: Checkout e2e test repo
64+
uses: actions/checkout@v3
65+
with:
66+
repository: cdapio/cdap-e2e-tests
67+
path: e2e
68+
69+
- name: Cache
70+
uses: actions/cache@v3
71+
with:
72+
path: ~/.m2/repository
73+
key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
74+
restore-keys: |
75+
${{ runner.os }}-maven-${{ github.workflow }}
76+
77+
- name: Get Secrets from GCP Secret Manager
78+
id: secrets
79+
uses: 'google-github-actions/get-secretmanager-secrets@v0'
80+
with:
81+
secrets: |-
82+
MYSQL_HOST:cdapio-github-builds/MYSQL_HOST
83+
MYSQL_USERNAME:cdapio-github-builds/MYSQL_USERNAME
84+
MYSQL_PASSWORD:cdapio-github-builds/MYSQL_PASSWORD
85+
MYSQL_PORT:cdapio-github-builds/MYSQL_PORT
86+
87+
- name: Run required e2e tests
88+
if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false'
89+
run: python3 e2e/src/main/scripts/run_e2e_test.py --mvnTestRunProfiles e2e-tests,skip-its --mvnProjectBuildProfiles skip-its --module ${{ matrix.module }} --testRunner TestRunnerRequired.java
90+
env:
91+
MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }}
92+
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
93+
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}
94+
MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }}
95+
96+
- name: Run all e2e tests
97+
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true'
98+
run: python3 e2e/src/main/scripts/run_e2e_test.py --mvnTestRunProfiles e2e-tests,skip-its --mvnProjectBuildProfiles skip-its --module ${{ matrix.module }}
99+
env:
100+
MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }}
101+
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
102+
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}
103+
MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }}
104+
105+
- name: Upload report
106+
uses: actions/upload-artifact@v3
107+
if: always()
108+
with:
109+
name: Cucumber report - ${{ matrix.module }}
110+
path: ./**/target/cucumber-reports
111+
112+
- name: Upload debug files
113+
uses: actions/upload-artifact@v3
114+
if: always()
115+
with:
116+
name: Debug files - ${{ matrix.module }}
117+
path: ./**/target/e2e-debug
118+
119+
- name: Upload files to GCS
120+
uses: google-github-actions/upload-cloud-storage@v0
121+
if: always()
122+
with:
123+
path: ./plugin
124+
destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}
125+
glob: '**/target/cucumber-reports/**'

‎mysql-delta-plugins/src/e2e-test/features/mysql/DesignTimeValidation.feature‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mysql @Required
18+
Feature: Mysql - Verify Mysql source plugin design time validation scenarios
19+
20+
Scenario: To verify validation message when user provides invalid Host
21+
Given Open DataFusion Project with replication to configure replication job
22+
When Enter input plugin property: "name" with pipelineName
23+
And Click on the "Next" button in replication to navigate
24+
And Select Source plugin: "MySQL" from the replication plugins list
25+
Then Replace input plugin property: "host" with value: "mySqlInvalidHost" for Credentials and Authorization related fields
26+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
27+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
28+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
29+
Then Replace input plugin property: "user" with value: "mysqlUsername" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "password" with value: "mysqlPassword" for Credentials and Authorization related fields
31+
And Click on the "Next" button in replication to navigate
32+
Then Replace input plugin property: "project" with value: "projectId"
33+
Then Enter input plugin property: "datasetName" with value: "dataset"
34+
And Click on the "Next" button in replication to navigate
35+
Then Verify that the Plugin is displaying an error message: "errorMessageMysqlInvalidHost"
36+
37+
Scenario: To verify validation message when user provides invalid Port
38+
Given Open DataFusion Project with replication to configure replication job
39+
When Enter input plugin property: "name" with pipelineName
40+
And Click on the "Next" button in replication to navigate
41+
And Select Source plugin: "MySQL" from the replication plugins list
42+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "mySqlInvalidPort" for Credentials and Authorization related fields
44+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
45+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
46+
Then Replace input plugin property: "user" with value: "mysqlUsername" for Credentials and Authorization related fields
47+
Then Replace input plugin property: "password" with value: "mysqlPassword" for Credentials and Authorization related fields
48+
And Click on the "Next" button in replication to navigate
49+
Then Replace input plugin property: "project" with value: "projectId"
50+
Then Enter input plugin property: "datasetName" with value: "dataset"
51+
And Click on the "Next" button in replication to navigate
52+
Then Verify that the Plugin is displaying an error message: "errorMessageMysqlInvalidPort"
53+
54+
Scenario: To verify validation message when user provides invalid Database Name
55+
Given Open DataFusion Project with replication to configure replication job
56+
When Enter input plugin property: "name" with pipelineName
57+
And Click on the "Next" button in replication to navigate
58+
And Select Source plugin: "MySQL" from the replication plugins list
59+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
60+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
61+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
62+
Then Replace input plugin property: "database" with value: "mySqlInvalidDatabase"
63+
Then Replace input plugin property: "user" with value: "mysqlUsername" for Credentials and Authorization related fields
64+
Then Replace input plugin property: "password" with value: "mysqlPassword" for Credentials and Authorization related fields
65+
And Click on the "Next" button in replication to navigate
66+
Then Replace input plugin property: "project" with value: "projectId"
67+
Then Enter input plugin property: "datasetName" with value: "dataset"
68+
And Click on the "Next" button in replication to navigate
69+
Then Verify that the Plugin is displaying an error message: "errorMessageMysqlInvalidDatabase"
70+
71+
Scenario: To verify validation message when user provides invalid user
72+
Given Open DataFusion Project with replication to configure replication job
73+
When Enter input plugin property: "name" with pipelineName
74+
And Click on the "Next" button in replication to navigate
75+
And Select Source plugin: "MySQL" from the replication plugins list
76+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
77+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
78+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
79+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
80+
Then Replace input plugin property: "user" with value: "mySqlInvalidUser" for Credentials and Authorization related fields
81+
Then Replace input plugin property: "password" with value: "mysqlPassword" for Credentials and Authorization related fields
82+
And Click on the "Next" button in replication to navigate
83+
Then Replace input plugin property: "project" with value: "projectId"
84+
Then Enter input plugin property: "datasetName" with value: "dataset"
85+
And Click on the "Next" button in replication to navigate
86+
Then Verify that the Plugin is displaying an error message: "errorMessageMysqlInvalidUser"
87+
88+
Scenario: To verify validation message when user provides invalid password
89+
Given Open DataFusion Project with replication to configure replication job
90+
When Enter input plugin property: "name" with pipelineName
91+
And Click on the "Next" button in replication to navigate
92+
And Select Source plugin: "MySQL" from the replication plugins list
93+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
94+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
95+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
96+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
97+
Then Replace input plugin property: "user" with value: "mySqlInvalidUser" for Credentials and Authorization related fields
98+
Then Replace input plugin property: "password" with value: "mySqlInvalidPassword" for Credentials and Authorization related fields
99+
And Click on the "Next" button in replication to navigate
100+
Then Replace input plugin property: "project" with value: "projectId"
101+
Then Enter input plugin property: "datasetName" with value: "dataset"
102+
And Click on the "Next" button in replication to navigate
103+
Then Verify that the Plugin is displaying an error message: "errorMessageMysqlInvalidPassword"
104+
105+
Scenario: To verify validation message when macro enabled for password field
106+
Given Open DataFusion Project with replication to configure replication job
107+
When Enter input plugin property: "name" with pipelineName
108+
And Click on the "Next" button in replication to navigate
109+
And Select Source plugin: "MySQL" from the replication plugins list
110+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
111+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
112+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
113+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
114+
Then Replace input plugin property: "user" with value: "mysqlUsername" for Credentials and Authorization related fields
115+
Then Click on the Macro button of Property: "password" and set the value to: "Password"
116+
And Click on the "Next" button in replication to navigate
117+
Then Replace input plugin property: "project" with value: "projectId"
118+
Then Enter input plugin property: "datasetName" with value: "dataset"
119+
And Click on the "Next" button in replication to navigate
120+
Then Verify that the Plugin is displaying an error message: "errorMessageMacroPassword"
121+
122+
Scenario: To verify validation message when macro enabled for service account key field
123+
Given Open DataFusion Project with replication to configure replication job
124+
When Enter input plugin property: "name" with pipelineName
125+
And Click on the "Next" button in replication to navigate
126+
And Select Source plugin: "MySQL" from the replication plugins list
127+
Then Replace input plugin property: "host" with value: "mysqlHost" for Credentials and Authorization related fields
128+
Then Replace input plugin property: "port" with value: "mysqlPort" for Credentials and Authorization related fields
129+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "mysqlDriverName"
130+
Then Replace input plugin property: "database" with value: "mysqlDatabaseName"
131+
Then Replace input plugin property: "user" with value: "mysqlUsername" for Credentials and Authorization related fields
132+
Then Replace input plugin property: "password" with value: "mysqlPassword" for Credentials and Authorization related fields
133+
And Click on the "Next" button in replication to navigate
134+
Then Replace input plugin property: "project" with value: "projectId"
135+
Then Click on the Macro button of Property: "serviceAccountKey" and set the value to: "ServiceAccountKey"
136+
Then Enter input plugin property: "datasetName" with value: "dataset"
137+
And Click on the "Next" button in replication to navigate
138+
Then Verify that the Plugin is displaying an error message: "errorMessageMacroServiceAccountKey"
139+

0 commit comments

Comments
 (0)