github-forks
/
flink-cdc-connectors
mirror of https://github.com/ververica/flink-cdc-connectors.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[cdc][e2e] Unify logic to clean docker image when connector e2e test finished (#2978)

Browse Source
pull/2994/head
gongzhongqiang 1 year ago committed by GitHub
parent f3762a1b30
commit a24c5d1b24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 68 deletions
Show Stats Download Patch File Download Diff File

2
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/com/ververica/cdc/connectors/mysql/testutils/UniqueDatabase.java
Unescape Escape View File

@ -142,7 +142,7 @@ public class UniqueDatabase {
}
}
/** Drop the database if is exists. */
/** Drop the database if it is existing. */
public void dropDatabase() {
try {
try (Connection connection =

18
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/Db2E2eITCase.java
Unescape Escape View File

@ -20,7 +20,6 @@ import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -114,23 +113,6 @@ public class Db2E2eITCase extends FlinkContainerTestEnvironment {
super.after();
}
@AfterClass
public static void afterClass() {
// Cleanup the db2 image, because it's too large and will cause the next test to fail.
db2Container.getDockerClient().removeImageCmd(DB2_CUSTOM_IMAGE).exec();
db2Container
.getDockerClient()
.listImagesCmd()
.withImageNameFilter(DB2_IMAGE)
.exec()
.forEach(
image ->
db2Container
.getDockerClient()
.removeImageCmd(image.getId())
.exec());
}
@Test
public void testDb2CDC() throws Exception {
List<String> sqlLines =

19
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OceanBaseE2eITCase.java
Unescape Escape View File

@ -16,17 +16,14 @@
package com.ververica.cdc.connectors.tests;
import com.github.dockerjava.api.DockerClient;
import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.Wait;
@ -46,7 +43,6 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertNotNull;
@ -116,21 +112,6 @@ public class OceanBaseE2eITCase extends FlinkContainerTestEnvironment {
}
}
@AfterClass
public static void afterClass() {
Stream.of(OB_SERVER, LOG_PROXY).forEach(GenericContainer::stop);
DockerClient client = DockerClientFactory.instance().client();
client.listImagesCmd()
.withImageNameFilter(OB_SERVER_IMAGE)
.exec()
.forEach(image -> client.removeImageCmd(image.getId()).exec());
client.listImagesCmd()
.withImageNameFilter(OB_LOG_PROXY_IMAGE)
.exec()
.forEach(image -> client.removeImageCmd(image.getId()).exec());
}
@Test
public void testOceanBaseCDC() throws Exception {
List<String> sqlLines =

11
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/OracleE2eITCase.java
Unescape Escape View File

@ -20,7 +20,6 @@ import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -84,16 +83,6 @@ public class OracleE2eITCase extends FlinkContainerTestEnvironment {
super.after();
}
@AfterClass
public static void afterClass() {
// Cleanup the oracle image, because it's too large and will cause the next test to fail.
oracle.getDockerClient()
.listImagesCmd()
.withImageNameFilter(ORACLE_IMAGE)
.exec()
.forEach(image -> oracle.getDockerClient().removeImageCmd(image.getId()).exec());
}
@Test
public void testOracleCDC() throws Exception {
List<String> sqlLines =

8
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/SqlServerE2eITCase.java
Unescape Escape View File

@ -20,14 +20,12 @@ import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
@ -103,12 +101,6 @@ public class SqlServerE2eITCase extends FlinkContainerTestEnvironment {
super.after();
}
@AfterClass
public static void afterClass() {
// Cleanup the sqlserver image, because it's too large and will cause the next test to fail.
DockerClientFactory.instance().client().removeImageCmd(MSSQL_SERVER_IMAGE).exec();
}
@Test
public void testSqlServerCDC() throws Exception {
List<String> sqlLines =

11
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/TiDBE2eITCase.java
Unescape Escape View File

@ -20,7 +20,6 @@ import com.ververica.cdc.connectors.tests.utils.FlinkContainerTestEnvironment;
import com.ververica.cdc.connectors.tests.utils.JdbcProxy;
import com.ververica.cdc.connectors.tests.utils.TestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
@ -142,16 +141,6 @@ public class TiDBE2eITCase extends FlinkContainerTestEnvironment {
super.after();
}
@AfterClass
public static void afterClass() {
// Cleanup the TIDB image, because it's too large and will cause the next test to fail.
TIDB.getDockerClient()
.listImagesCmd()
.withImageNameFilter("pingcap/tikv")
.exec()
.forEach(image -> TIDB.getDockerClient().removeImageCmd(image.getId()).exec());
}
@Test
public void testTIDBCDC() throws Exception {
List<String> sqlLines =

42
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/com/ververica/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
Unescape Escape View File

@ -26,10 +26,12 @@ import org.apache.flink.runtime.client.JobStatusMessage;
import org.apache.flink.table.api.ValidationException;
import org.apache.flink.util.TestLogger;
import com.github.dockerjava.api.DockerClient;
import com.ververica.cdc.connectors.mysql.testutils.MySqlContainer;
import com.ververica.cdc.connectors.mysql.testutils.MySqlVersion;
import com.ververica.cdc.connectors.mysql.testutils.UniqueDatabase;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
@ -38,6 +40,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.Container.ExecResult;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
@ -174,6 +177,45 @@ public abstract class FlinkContainerTestEnvironment extends TestLogger {
mysqlInventoryDatabase.dropDatabase();
}
@AfterClass
public static void afterClass() {
DockerClient dockerClient = DockerClientFactory.instance().client();
// List all containers and remove the ones that are not testcontainers related.
dockerClient.listContainersCmd().exec().stream()
.filter(container -> !container.getImage().startsWith("testcontainers"))
.forEach(
container -> {
dockerClient.stopContainerCmd(container.getId()).exec();
dockerClient.removeContainerCmd(container.getId()).exec();
});
// List all images and remove the ones that are not flink、mysql、testcontainers related.
dockerClient.listImagesCmd().exec().stream()
.filter(
image ->
image.getRepoTags() != null
&& Arrays.stream(image.getRepoTags())
.anyMatch(
tag ->
!tag.startsWith("flink:")
&& !tag.startsWith(
"testcontainers")
&& !tag.equals(
MYSQL
.getDockerImageName())))
.forEach(
image -> {
try {
dockerClient.removeImageCmd(image.getId()).exec();
} catch (Exception e) {
LOG.warn(
"Failed to remove image: {}",
String.join(",", image.getRepoTags()));
}
});
}
/** Allow overriding the default flink properties. */
public void overrideFlinkProperties(String properties) {
jobManager.withEnv("FLINK_PROPERTIES", properties);

Write Preview
Loading…
Cancel
Save