Skip to content

Commit bb70fa3

Browse files
Al-assadwolfboys
Al-assad
and
wolfboys
authored
Add unit tests for DockerTool (apache#276)
* add .gitignore * build fat-jar when submit flink k8s-session job * remove gitkeep * config const for k8s and docker environment * import docker-client dependencies * add docker operator tool * add tryWithResource support * add default image namespace const * add default image namespace const * fix bug * add default image namespace const * change spec out path and add it to .gitignore * add unit test * remove unnecessary try-with-resource * add unit test for DockerTools and bug fix * Update DockerTool.scala Co-authored-by: benjobs <[email protected]>
1 parent 882642b commit bb70fa3

File tree

7 files changed

+173
-25
lines changed

7 files changed

+173
-25
lines changed

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -384,4 +384,6 @@ $RECYCLE.BIN/
384384
# Windows shortcuts
385385
*.lnk
386386

387-
MavenToolTest-output/
387+
# unit test temporary output directory
388+
*Spec-output/
389+

‎streamx-codebuild/src/main/scala/com/streamxhub/streamx/codebuild/DockerRetriver.scala

+1-3
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@ object DockerRetriver {
4040
* todo support custom docker configuration parameters in unifined configurations in the future
4141
*/
4242
lazy val dockerClientConf: DockerClientConfig = {
43-
DefaultDockerClientConfig.createDefaultConfigBuilder().
44-
build()
43+
DefaultDockerClientConfig.createDefaultConfigBuilder().build()
4544
}
4645

4746
/**
@@ -74,5 +73,4 @@ object DockerRetriver {
7473
def newDockerClient(): DockerClient = DockerClientImpl.getInstance(dockerClientConf, dockerHttpClient)
7574

7675

77-
7876
}

‎streamx-codebuild/src/main/scala/com/streamxhub/streamx/codebuild/DockerTool.scala

+22-17
Original file line numberDiff line numberDiff line change
@@ -52,20 +52,23 @@ object DockerTool {
5252
}
5353
val flinkFatJar = new File(template.flinkFatjarPath)
5454
if (flinkFatJar.getParentFile.getAbsolutePath != projectDir.getAbsolutePath) {
55-
FileUtils.copyFile(flinkFatJar, projectDir)
55+
FileUtils.copyFile(flinkFatJar, new File(s"${projectDir.getAbsolutePath}/${flinkFatJar.getName}"))
5656
}
5757
val dockerfile = template.writeDockerfile(projectPath)
5858
// build and push docker image
5959
val tagName = buildImage(projectDir, dockerfile, tag)
6060
if (push) {
6161
pushImage(tagName)
62+
} else {
63+
tagName
6264
}
63-
tagName
65+
6466
}
6567

6668
/**
6769
* build docker image.
68-
* this sync call api.
70+
* this is sync call api.
71+
*
6972
* @param baseDir base directory
7073
* @param dockerfile dockerfile
7174
* @param tag image tag
@@ -75,38 +78,39 @@ object DockerTool {
7578
def buildImage(baseDir: File, dockerfile: File, tag: String): String = {
7679
val tagName = compileTag(tag)
7780
tryWithResource(DockerRetriver.newDockerClient()) {
78-
client =>
81+
client => {
7982
// build docker image
8083
val buildImageCmd = client.buildImageCmd()
84+
.withPull(true)
8185
.withBaseDirectory(baseDir)
8286
.withDockerfile(dockerfile)
8387
.withTags(Sets.newHashSet(tagName))
84-
tryWithResource(buildImageCmd.start()) {
85-
result => result.awaitCompletion()
86-
}
88+
buildImageCmd.start().awaitCompletion()
89+
}
8790
}
8891
tagName
8992
}
9093

9194
/**
9295
* push image to remote repository.
93-
* this sync call api.
96+
* this is sync call api.
97+
*
98+
* @param tag tag name
99+
* @return actually image tag
94100
*/
95-
def pushImage(tag: String): Unit = {
96-
val tageName = compileTag(tag)
101+
def pushImage(tag: String): String = {
102+
val tagName = compileTag(tag)
97103
tryWithResource(DockerRetriver.newDockerClient()) {
98104
client => {
99-
val pushCmd: PushImageCmd = client.pushImageCmd(tageName)
105+
val pushCmd: PushImageCmd = client.pushImageCmd(tagName)
100106
.withAuthConfig(DockerRetriver.remoteImageRegisterAuthConfig)
101-
.withTag(tageName)
102-
tryWithResource(pushCmd.start()) {
103-
result => result.awaitCompletion()
104-
}
107+
.withTag(tagName)
108+
pushCmd.start().awaitCompletion()
105109
}
106110
}
111+
tagName
107112
}
108113

109-
110114
/**
111115
* compile image tag with namespace and remote address.
112116
*/
@@ -116,8 +120,9 @@ object DockerTool {
116120
else s"$IMAGE_NAMESPACE/$tag"
117121
}
118122
if (K8S_IMAGE_REGISTER_ADDRESS.nonEmpty && !tagName.startsWith(K8S_IMAGE_REGISTER_ADDRESS)) {
119-
tagName = s"$K8S_IMAGE_REGISTER_ADDRESS/$tagName"
123+
tagName = s"$K8S_IMAGE_REGISTER_ADDRESS/$tagName"
120124
}
125+
tagName
121126
}
122127

123128

‎streamx-codebuild/src/test/scala/com/streamxhub/streamx/codebuild/DockerToolSpec.scala

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Copyright (c) 2021 The StreamX Project
3+
* <p>
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
* <p>
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
* <p>
14+
* Unless required by applicable law or agreed to in writing,
15+
* software distributed under the License is distributed on an
16+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
* KIND, either express or implied. See the License for the
18+
* specific language governing permissions and limitations
19+
* under the License.
20+
*/
21+
package com.streamxhub.streamx.codebuild
22+
23+
import com.streamxhub.streamx.common.conf.K8sConfigConst.{IMAGE_NAMESPACE, K8S_IMAGE_REGISTER_ADDRESS, KEY_K8S_IMAGE_REGISTER_ADDRESS}
24+
import org.apache.commons.io.FileUtils
25+
import org.scalatest.BeforeAndAfter
26+
import org.scalatest.matchers.must.Matchers
27+
import org.scalatest.wordspec.AnyWordSpec
28+
29+
import java.io.File
30+
31+
class DockerToolSpec extends AnyWordSpec with BeforeAndAfter with Matchers {
32+
33+
val outputDir = new File("DockerToolSpec-output/")
34+
35+
before {
36+
outputDir.mkdir()
37+
}
38+
39+
after {
40+
FileUtils.forceDelete(outputDir)
41+
}
42+
43+
"DockerTool" when {
44+
"build flink image without push" should {
45+
"when remote register is not set" in {
46+
val template = new FlinkDockerfileTemplate("flink:1.13.0-scala_2.11", path("flink/WordCountSQL.jar"))
47+
val tag = DockerTool.buildFlinkImage(outputDir.getAbsolutePath, template, "myflink-job")
48+
tag mustBe s"$IMAGE_NAMESPACE/myflink-job"
49+
}
50+
"when remote register is set" in {
51+
System.setProperty(KEY_K8S_IMAGE_REGISTER_ADDRESS, "registry.cn-hangzhou.aliyuncs.com")
52+
val template = new FlinkDockerfileTemplate("flink:1.13.0-scala_2.11", path("flink/WordCountSQL.jar"))
53+
val tag = DockerTool.buildFlinkImage(outputDir.getAbsolutePath, template, "myflink-job")
54+
tag mustBe s"$K8S_IMAGE_REGISTER_ADDRESS/$IMAGE_NAMESPACE/myflink-job"
55+
56+
}
57+
}
58+
"operate image" should {
59+
"build image" ignore {}
60+
"push image" ignore {}
61+
}
62+
63+
}
64+
65+
}
66+

‎streamx-codebuild/src/test/scala/com/streamxhub/streamx/codebuild/FlinkDockerfileTemplateSpec.scala

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright (c) 2021 The StreamX Project
3+
* <p>
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
* <p>
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
* <p>
14+
* Unless required by applicable law or agreed to in writing,
15+
* software distributed under the License is distributed on an
16+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
* KIND, either express or implied. See the License for the
18+
* specific language governing permissions and limitations
19+
* under the License.
20+
*/
21+
package com.streamxhub.streamx.codebuild
22+
23+
import com.streamxhub.streamx.codebuild.FlinkDockerfileTemplate.DEFAULT_DOCKER_FILE_NAME
24+
import org.apache.commons.io.FileUtils
25+
import org.scalatest.BeforeAndAfter
26+
import org.scalatest.matchers.must.Matchers
27+
import org.scalatest.wordspec.AnyWordSpec
28+
29+
import java.io.File
30+
31+
class FlinkDockerfileTemplateSpec extends AnyWordSpec with BeforeAndAfter with Matchers {
32+
33+
val outputDir = new File("FlinkDockerfileTemplateSpec-output/")
34+
35+
val assertDockerFileContent =
36+
"""FROM 1.13-scala_2.11
37+
|RUN mkdir -p $FLINK_HOME/usrlib
38+
|COPY /WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar
39+
|""".stripMargin
40+
41+
before {
42+
outputDir.mkdir()
43+
}
44+
after {
45+
FileUtils.forceDelete(outputDir)
46+
}
47+
48+
"FlinkDockerfileTemplate" when {
49+
50+
"create dockerfile" should {
51+
val template = new FlinkDockerfileTemplate("1.13-scala_2.11", path("flink/WordCountSQL.jar"))
52+
val assertDockerFileContent =
53+
"""FROM 1.13-scala_2.11
54+
|RUN mkdir -p $FLINK_HOME/usrlib
55+
|COPY /WordCountSQL.jar $FLINK_HOME/usrlib/WordCountSQL.jar
56+
|""".stripMargin
57+
58+
"build Dockerfile content" in {
59+
template.dockerfileContent mustBe assertDockerFileContent
60+
}
61+
"write Dockerfile to file" in {
62+
val output = outputDir.getAbsolutePath + "/my-dockerfile"
63+
val outFile = template.writeDockerfile(output)
64+
FileUtils.readFileToString(outFile, "UTF-8") mustBe assertDockerFileContent
65+
}
66+
"write Dockerfile to directory" in {
67+
val outFile = template.writeDockerfile(outputDir.getAbsolutePath)
68+
outFile.getName mustBe DEFAULT_DOCKER_FILE_NAME
69+
FileUtils.readFileToString(outFile, "UTF-8") mustBe assertDockerFileContent
70+
}
71+
}
72+
73+
}
74+
75+
}

‎streamx-codebuild/src/test/scala/com/streamxhub/streamx/codebuild/MavenToolSpec.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import scala.language.postfixOps
3333

3434
class MavenToolSpec extends AnyWordSpec with BeforeAndAfterAll with Matchers {
3535

36-
val outputDir = "MavenToolTest-output/"
36+
val outputDir = "MavenToolSpec-output/"
3737
val preWorkSpaceVal: String = System.getProperties.getProperty(KEY_STREAMX_WORKSPACE)
3838

3939
override protected def beforeAll(): Unit = {

‎streamx-common/src/main/scala/com/streamxhub/streamx/common/conf/K8sConfigConst.scala

+5-3
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ package com.streamxhub.streamx.common.conf
2727
object K8sConfigConst {
2828

2929
/**
30-
* docker image regoster address for remote k8s cluster
30+
* docker image regoster address for remote k8s cluster.
31+
* when this configuration item is empty, it means that
32+
* the dockerhub public repository is used.
3133
*/
3234
lazy val KEY_K8S_IMAGE_REGISTER_ADDRESS = "k8s.image.register.address"
3335
lazy val K8S_IMAGE_REGISTER_ADDRESS: String = System.getProperty(KEY_K8S_IMAGE_REGISTER_ADDRESS, "")
@@ -36,13 +38,13 @@ object K8sConfigConst {
3638
* login username of docker image regoster for remote k8s cluster
3739
*/
3840
lazy val KEY_K8S_IMAGE_REGISTER_USERNAME = "k8s.image.register.username"
39-
lazy val K8S_IMAGE_REGISTER_USERNAME: String = System.getProperty(KEY_K8S_IMAGE_REGISTER_ADDRESS, "")
41+
lazy val K8S_IMAGE_REGISTER_USERNAME: String = System.getProperty(KEY_K8S_IMAGE_REGISTER_USERNAME, "")
4042

4143
/**
4244
* login password of docker image regoster for remote k8s cluster
4345
*/
4446
lazy val KEY_K8S_IMAGE_REGISTER_PASSWORD = "k8s.image.register.password"
45-
lazy val K8S_IMAGE_REGISTER_PASSWORD: String = System.getProperty(KEY_K8S_IMAGE_REGISTER_ADDRESS, "")
47+
lazy val K8S_IMAGE_REGISTER_PASSWORD: String = System.getProperty(KEY_K8S_IMAGE_REGISTER_PASSWORD, "")
4648

4749
/**
4850
* namespace for docker image used in docker build env and image register

0 commit comments

Comments
 (0)