parent
b15a226ccd
commit
5917f78704
@ -0,0 +1,8 @@
|
|||||||
|
*.sql
|
||||||
|
savepoints/**
|
||||||
|
cdc-versions/**
|
||||||
|
cache/**
|
||||||
|
.idea/**
|
||||||
|
Gemfile.lock
|
||||||
|
/logs/
|
||||||
|
conf/temp.yaml
|
@ -0,0 +1,38 @@
|
|||||||
|
# Flink CDC Migration Test Utilities
|
||||||
|
|
||||||
|
## Pipeline Jobs
|
||||||
|
### Preparation
|
||||||
|
|
||||||
|
1. Install Ruby (macOS has embedded it by default)
|
||||||
|
2. (Optional) Run `gem install terminal-table` for better display
|
||||||
|
|
||||||
|
### Compile snapshot CDC versions
|
||||||
|
3. Set `CDC_SOURCE_HOME` to the root directory of the Flink CDC git repository
|
||||||
|
4. Go to `tools/mig-test` and run `ruby prepare_libs.rb` to download released / compile snapshot CDC versions
|
||||||
|
|
||||||
|
### Run migration tests
|
||||||
|
5. Enter `conf/` and run `docker compose up -d` to start up test containers
|
||||||
|
6. Set `FLINK_HOME` to the home directory of Flink
|
||||||
|
7. Go back to `tools/mig-test` and run `ruby run_migration_test.rb` to start testing
|
||||||
|
|
||||||
|
### Result
|
||||||
|
The migration result will be displayed in the console like this:
|
||||||
|
|
||||||
|
```
|
||||||
|
+--------------------------------------------------------------------+
|
||||||
|
| Migration Test Result |
|
||||||
|
+--------------+-------+-------+-------+--------------+--------------+
|
||||||
|
| | 3.0.0 | 3.0.1 | 3.1.0 | 3.1-SNAPSHOT | 3.2-SNAPSHOT |
|
||||||
|
| 3.0.0 | ❓ | ❓ | ❌ | ✅ | ✅ |
|
||||||
|
| 3.0.1 | | ❓ | ❌ | ✅ | ✅ |
|
||||||
|
| 3.1.0 | | | ✅ | ❌ | ❌ |
|
||||||
|
| 3.1-SNAPSHOT | | | | ✅ | ✅ |
|
||||||
|
| 3.2-SNAPSHOT | | | | | ✅ |
|
||||||
|
+--------------+-------+-------+-------+--------------+--------------+
|
||||||
|
```
|
||||||
|
|
||||||
|
> ✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`
|
||||||
|
|
||||||
|
## DataStream Jobs
|
||||||
|
|
||||||
|
See `datastream/README.md`.
|
@ -0,0 +1,24 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
mysql:
|
||||||
|
image: mysql:8.0
|
||||||
|
environment:
|
||||||
|
MYSQL_ALLOW_EMPTY_PASSWORD: "true"
|
||||||
|
MYSQL_DATABASE: fallen
|
||||||
|
ports:
|
||||||
|
- 3306:3306
|
@ -0,0 +1,41 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
source:
|
||||||
|
type: mysql
|
||||||
|
name: MySQL Source
|
||||||
|
hostname: localhost
|
||||||
|
port: 3306
|
||||||
|
username: root
|
||||||
|
password: ""
|
||||||
|
tables: fallen.\.*
|
||||||
|
server-id: 5400-5500
|
||||||
|
server-time-zone: UTC
|
||||||
|
|
||||||
|
sink:
|
||||||
|
type: values
|
||||||
|
name: Values Sink
|
||||||
|
|
||||||
|
pipeline:
|
||||||
|
name: ${PIPELINE_NAME}
|
||||||
|
parallelism: 1
|
||||||
|
|
||||||
|
transform:
|
||||||
|
- source-table: fallen.\.*
|
||||||
|
projection: \*, 'extras' AS EXTRAS
|
||||||
|
|
||||||
|
route:
|
||||||
|
- source-table: fallen.\.*
|
||||||
|
sink-table: fallen.terminus
|
@ -0,0 +1,33 @@
|
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
source:
|
||||||
|
type: mysql
|
||||||
|
name: MySQL Source
|
||||||
|
hostname: localhost
|
||||||
|
port: 3306
|
||||||
|
username: root
|
||||||
|
password: ""
|
||||||
|
tables: fallen.\.*
|
||||||
|
server-id: 5400-5500
|
||||||
|
server-time-zone: UTC
|
||||||
|
|
||||||
|
sink:
|
||||||
|
type: values
|
||||||
|
name: Values Sink
|
||||||
|
|
||||||
|
pipeline:
|
||||||
|
name: ${PIPELINE_NAME}
|
||||||
|
parallelism: 1
|
@ -0,0 +1,11 @@
|
|||||||
|
# Flink CDC MigrationTestUtils
|
||||||
|
|
||||||
|
## DataStream Jobs
|
||||||
|
### Preparation
|
||||||
|
|
||||||
|
1. Install Ruby (macOS has embedded it by default)
|
||||||
|
2. (Optional) Run `gem install terminal-table` for better display
|
||||||
|
|
||||||
|
### Compile DataStream Jobs
|
||||||
|
3. Go to `tools/mig-test/datastream` and run `ruby compile_jobs.rb` to compile dummy DataStream jobs with specific version tags
|
||||||
|
4. Then, run `ruby run_migration_test.rb` to start testing
|
@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
JOB_VERSIONS = %w[2.4.2 3.0.0 3.0.1 3.1.0 3.1.1 3.2-SNAPSHOT]
|
||||||
|
|
||||||
|
JOB_VERSIONS.each do |version|
|
||||||
|
puts "Compiling DataStream job for CDC #{version}"
|
||||||
|
`cd datastream-#{version} && mvn clean package -DskipTests`
|
||||||
|
end
|
||||||
|
|
||||||
|
puts 'Done'
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>2.4.2</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.17.1</flink.version>
|
||||||
|
<flink.cdc.version>2.4.2</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>30.1.1-jre-16.1</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>3.0.0</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.18.1</flink.version>
|
||||||
|
<flink.cdc.version>3.0.0</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>31.1-jre-17.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>3.0.1</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.18.1</flink.version>
|
||||||
|
<flink.cdc.version>3.0.1</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>31.1-jre-17.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.ververica</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import com.ververica.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>3.1.0</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.18.1</flink.version>
|
||||||
|
<flink.cdc.version>3.1.0</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>31.1-jre-17.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>3.1.1</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.18.1</flink.version>
|
||||||
|
<flink.cdc.version>3.1.1</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>31.1-jre-17.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
@ -0,0 +1,151 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>datastream-job</artifactId>
|
||||||
|
<version>3.2-SNAPSHOT</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.18.1</flink.version>
|
||||||
|
<flink.cdc.version>3.2-SNAPSHOT</flink.cdc.version>
|
||||||
|
<debezium.version>1.9.7.Final</debezium.version>
|
||||||
|
<scala.binary.version>2.12</scala.binary.version>
|
||||||
|
<slf4j.version>2.0.13</slf4j.version>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-base</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-clients</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-runtime</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-core</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-table-common</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- Checked the dependencies of the Flink project and below is a feasible reference. -->
|
||||||
|
<!-- Use flink shaded guava 18.0-13.0 for flink 1.13 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-14.0 for flink-1.14 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.15 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-15.0 for flink-1.16 -->
|
||||||
|
<!-- Use flink shaded guava 30.1.1-jre-16.1 for flink-1.17 -->
|
||||||
|
<!-- Use flink shaded guava 31.1-jre-17.0 for flink-1.18 -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-shaded-guava</artifactId>
|
||||||
|
<version>31.1-jre-17.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-debezium</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-cdc-base</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-mysql-cdc</artifactId>
|
||||||
|
<version>${flink.cdc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.debezium</groupId>
|
||||||
|
<artifactId>debezium-connector-mysql</artifactId>
|
||||||
|
<version>${debezium.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- any other plugins -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
@ -0,0 +1,54 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.source.MySqlSource;
|
||||||
|
import org.apache.flink.cdc.connectors.mysql.table.StartupOptions;
|
||||||
|
import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema;
|
||||||
|
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
public class DataStreamJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
|
||||||
|
.hostname("localhost")
|
||||||
|
.port(3306)
|
||||||
|
.databaseList("fallen")
|
||||||
|
.tableList("fallen.angel", "fallen.gabriel", "fallen.girl")
|
||||||
|
.startupOptions(StartupOptions.initial())
|
||||||
|
.username("root")
|
||||||
|
.password("")
|
||||||
|
.deserializer(new JsonDebeziumDeserializationSchema())
|
||||||
|
.serverTimeZone("UTC")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
env.enableCheckpointing(3000);
|
||||||
|
|
||||||
|
env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source")
|
||||||
|
.uid("sql-source-uid")
|
||||||
|
.setParallelism(1)
|
||||||
|
.print()
|
||||||
|
.setParallelism(1);
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
// ... unfortunately
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,138 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
require 'pathname'
|
||||||
|
require 'securerandom'
|
||||||
|
|
||||||
|
WAITING_SECONDS = 20
|
||||||
|
FLINK_HOME = ENV['FLINK_HOME']
|
||||||
|
throw 'Unspecified `FLINK_HOME` environment variable.' if FLINK_HOME.nil?
|
||||||
|
FLINK_HOME = Pathname.new(FLINK_HOME).realpath
|
||||||
|
|
||||||
|
SOURCE_PORT = 3306
|
||||||
|
DATABASE_NAME = 'fallen'
|
||||||
|
TABLES = ['girl'].freeze
|
||||||
|
|
||||||
|
def exec_sql_source(sql)
|
||||||
|
`mysql -h 127.0.0.1 -P#{SOURCE_PORT} -uroot --skip-password -e "USE #{DATABASE_NAME}; #{sql}"`
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_mystery_data(mystery)
|
||||||
|
exec_sql_source("REPLACE INTO girl(id, name) VALUES (17, '#{mystery}');")
|
||||||
|
end
|
||||||
|
|
||||||
|
def ensure_mystery_data(mystery)
|
||||||
|
throw StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery
|
||||||
|
end
|
||||||
|
|
||||||
|
puts ' Waiting for source to start up...'
|
||||||
|
next until exec_sql_source("SELECT '1';") == "1\n1\n"
|
||||||
|
|
||||||
|
def test_migration_chore(from_version, to_version)
|
||||||
|
TABLES.each do |table_name|
|
||||||
|
exec_sql_source("DROP TABLE IF EXISTS #{table_name};")
|
||||||
|
exec_sql_source("CREATE TABLE #{table_name} (ID INT NOT NULL, NAME VARCHAR(17), PRIMARY KEY (ID));")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Clear previous savepoints and logs
|
||||||
|
`rm -rf savepoints`
|
||||||
|
|
||||||
|
old_job_id = `#{FLINK_HOME}/bin/flink run -p 1 -c DataStreamJob --detached datastream-#{from_version}/target/datastream-job-#{from_version}-jar-with-dependencies.jar`.split.last
|
||||||
|
raise StandardError, 'Failed to submit Flink job' unless old_job_id.length == 32
|
||||||
|
|
||||||
|
puts "Submitted job at #{from_version} as #{old_job_id}"
|
||||||
|
|
||||||
|
random_string_1 = SecureRandom.hex(8)
|
||||||
|
put_mystery_data random_string_1
|
||||||
|
sleep WAITING_SECONDS
|
||||||
|
ensure_mystery_data random_string_1
|
||||||
|
|
||||||
|
puts `#{FLINK_HOME}/bin/flink stop --savepointPath #{Dir.pwd}/savepoints #{old_job_id}`
|
||||||
|
savepoint_file = `ls savepoints`.split("\n").last
|
||||||
|
new_job_id = `#{FLINK_HOME}/bin/flink run --fromSavepoint #{Dir.pwd}/savepoints/#{savepoint_file} -p 1 -c DataStreamJob --detached datastream-#{to_version}/target/datastream-job-#{to_version}-jar-with-dependencies.jar`.split.last
|
||||||
|
raise StandardError, 'Failed to submit Flink job' unless new_job_id.length == 32
|
||||||
|
|
||||||
|
puts "Submitted job at #{to_version} as #{new_job_id}"
|
||||||
|
random_string_2 = SecureRandom.hex(8)
|
||||||
|
put_mystery_data random_string_2
|
||||||
|
sleep WAITING_SECONDS
|
||||||
|
ensure_mystery_data random_string_2
|
||||||
|
puts `#{FLINK_HOME}/bin/flink cancel #{new_job_id}`
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_migration(from_version, to_version)
|
||||||
|
puts "➡️ [MIGRATION] Testing migration from #{from_version} to #{to_version}..."
|
||||||
|
puts " with Flink #{FLINK_HOME}..."
|
||||||
|
begin
|
||||||
|
result = test_migration_chore from_version, to_version
|
||||||
|
if result
|
||||||
|
puts "✅ [MIGRATION] Successfully migrated from #{from_version} to #{to_version}!"
|
||||||
|
else
|
||||||
|
puts "❌ [MIGRATION] Failed to migrate from #{from_version} to #{to_version}..."
|
||||||
|
end
|
||||||
|
result
|
||||||
|
rescue => e
|
||||||
|
puts "❌ [MIGRATION] Failed to migrate from #{from_version} to #{to_version}...", e
|
||||||
|
false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
version_list = %w[2.4.2 3.0.0 3.0.1 3.1.0 3.1.1 3.2-SNAPSHOT]
|
||||||
|
version_result = Hash.new('❓')
|
||||||
|
@failures = []
|
||||||
|
|
||||||
|
version_list.each_with_index do |old_version, old_index|
|
||||||
|
puts 'Restarting cluster...'
|
||||||
|
`#{FLINK_HOME}/bin/stop-cluster.sh`
|
||||||
|
`rm -rf #{FLINK_HOME}/log/flink-*.out`
|
||||||
|
`#{FLINK_HOME}/bin/start-cluster.sh`
|
||||||
|
version_list.each_with_index do |new_version, new_index|
|
||||||
|
next if old_index > new_index
|
||||||
|
|
||||||
|
result = test_migration old_version, new_version
|
||||||
|
version_result[old_version + new_version] = result ? '✅' : '❌'
|
||||||
|
@failures << [old_version, new_version] unless result
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
printable_result = []
|
||||||
|
printable_result << [''] + version_list
|
||||||
|
version_list.each_with_index do |old_version, old_index|
|
||||||
|
table_line = [old_version]
|
||||||
|
version_list.each_with_index do |new_version, new_index|
|
||||||
|
table_line << if old_index > new_index
|
||||||
|
''
|
||||||
|
else
|
||||||
|
version_result[old_version + new_version]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
printable_result << table_line
|
||||||
|
end
|
||||||
|
|
||||||
|
begin
|
||||||
|
require 'terminal-table'
|
||||||
|
puts Terminal::Table.new rows: printable_result, title: 'Migration Test Result'
|
||||||
|
rescue LoadError
|
||||||
|
puts 'Test summary: ', printable_result
|
||||||
|
end
|
||||||
|
puts "✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`"
|
||||||
|
|
||||||
|
if @failures.filter { |_, new_version| new_version == version_list.last }.any?
|
||||||
|
abort 'Some migration to snapshot version tests failed.'
|
||||||
|
end
|
@ -0,0 +1,32 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
FLINK_HOME = ENV['FLINK_HOME']
|
||||||
|
throw 'Unspecified `FLINK_HOME` environment variable.' if FLINK_HOME.nil?
|
||||||
|
|
||||||
|
EXTRA_CONF = <<~EXTRACONF
|
||||||
|
|
||||||
|
taskmanager.numberOfTaskSlots: 10
|
||||||
|
parallelism.default: 4
|
||||||
|
execution.checkpointing.interval: 300
|
||||||
|
EXTRACONF
|
||||||
|
|
||||||
|
File.write("#{FLINK_HOME}/conf/flink-conf.yaml", EXTRA_CONF, mode: 'a+')
|
||||||
|
|
||||||
|
# MySQL connector is not provided
|
||||||
|
`wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar -O #{FLINK_HOME}/lib/mysql-connector-java-8.0.27.jar`
|
@ -0,0 +1,123 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
CDC_SOURCE_HOME = ENV['CDC_SOURCE_HOME']
|
||||||
|
throw 'Unspecified `CDC_SOURCE_HOME` environment variable.' if CDC_SOURCE_HOME.nil?
|
||||||
|
|
||||||
|
Dir.chdir(__dir__)
|
||||||
|
|
||||||
|
RELEASED_VERSIONS = {
|
||||||
|
'3.0.0': {
|
||||||
|
tar: 'https://github.com/ververica/flink-cdc-connectors/releases/download/release-3.0.0/flink-cdc-3.0.0-bin.tar.gz',
|
||||||
|
connectors: %w[
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-doris/3.0.0/flink-cdc-pipeline-connector-doris-3.0.0.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-mysql/3.0.0/flink-cdc-pipeline-connector-mysql-3.0.0.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-starrocks/3.0.0/flink-cdc-pipeline-connector-starrocks-3.0.0.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-values/3.0.0/flink-cdc-pipeline-connector-values-3.0.0.jar
|
||||||
|
]
|
||||||
|
},
|
||||||
|
'3.0.1': {
|
||||||
|
tar: 'https://github.com/ververica/flink-cdc-connectors/releases/download/release-3.0.1/flink-cdc-3.0.1-bin.tar.gz',
|
||||||
|
connectors: %w[
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-doris/3.0.1/flink-cdc-pipeline-connector-doris-3.0.1.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-mysql/3.0.1/flink-cdc-pipeline-connector-mysql-3.0.1.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-starrocks/3.0.1/flink-cdc-pipeline-connector-starrocks-3.0.1.jar
|
||||||
|
https://repo1.maven.org/maven2/com/ververica/flink-cdc-pipeline-connector-values/3.0.1/flink-cdc-pipeline-connector-values-3.0.1.jar
|
||||||
|
]
|
||||||
|
},
|
||||||
|
'3.1.0': {
|
||||||
|
tar: 'https://dlcdn.apache.org/flink/flink-cdc-3.1.0/flink-cdc-3.1.0-bin.tar.gz',
|
||||||
|
connectors: %w[
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-mysql/3.1.0/flink-cdc-pipeline-connector-mysql-3.1.0.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-doris/3.1.0/flink-cdc-pipeline-connector-doris-3.1.0.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-starrocks/3.1.0/flink-cdc-pipeline-connector-starrocks-3.1.0.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-kafka/3.1.0/flink-cdc-pipeline-connector-kafka-3.1.0.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-paimon/3.1.0/flink-cdc-pipeline-connector-paimon-3.1.0.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-values/3.1.0/flink-cdc-pipeline-connector-values-3.1.0.jar
|
||||||
|
]
|
||||||
|
},
|
||||||
|
'3.1.1': {
|
||||||
|
tar: 'https://dlcdn.apache.org/flink/flink-cdc-3.1.1/flink-cdc-3.1.1-bin.tar.gz',
|
||||||
|
connectors: %w[
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-mysql/3.1.1/flink-cdc-pipeline-connector-mysql-3.1.1.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-doris/3.1.1/flink-cdc-pipeline-connector-doris-3.1.1.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-starrocks/3.1.1/flink-cdc-pipeline-connector-starrocks-3.1.1.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-kafka/3.1.1/flink-cdc-pipeline-connector-kafka-3.1.1.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-paimon/3.1.1/flink-cdc-pipeline-connector-paimon-3.1.1.jar
|
||||||
|
https://repo1.maven.org/maven2/org/apache/flink/flink-cdc-pipeline-connector-values/3.1.1/flink-cdc-pipeline-connector-values-3.1.1.jar
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}.freeze
|
||||||
|
|
||||||
|
HEAD_VERSION = '3.2-SNAPSHOT'
|
||||||
|
|
||||||
|
def download_or_get(link)
|
||||||
|
`mkdir -p cache`
|
||||||
|
file_name = "cache/#{File.basename(link)}"
|
||||||
|
if File.exist? file_name
|
||||||
|
puts "#{file_name} exists, skip download"
|
||||||
|
return file_name
|
||||||
|
end
|
||||||
|
`wget #{link} -O #{file_name}`
|
||||||
|
file_name
|
||||||
|
end
|
||||||
|
|
||||||
|
M2_REPO = '~/.m2/repository/org/apache/flink'
|
||||||
|
FILES = %w[
|
||||||
|
dist
|
||||||
|
pipeline-connector-kafka
|
||||||
|
pipeline-connector-mysql
|
||||||
|
pipeline-connector-doris
|
||||||
|
pipeline-connector-paimon
|
||||||
|
pipeline-connector-starrocks
|
||||||
|
pipeline-connector-values
|
||||||
|
].freeze
|
||||||
|
def download_released
|
||||||
|
`rm -rf cdc-versions`
|
||||||
|
RELEASED_VERSIONS.each do |version, links|
|
||||||
|
`mkdir -p cdc-versions/#{version}`
|
||||||
|
file_name = download_or_get(links[:tar])
|
||||||
|
`tar --strip-components=1 -xzvf #{file_name} -C cdc-versions/#{version}`
|
||||||
|
links[:connectors].each do |link|
|
||||||
|
jar_file_name = download_or_get(link)
|
||||||
|
`cp #{jar_file_name} cdc-versions/#{version}/lib/`
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def compile_snapshot(version)
|
||||||
|
puts "Trying to create #{version}"
|
||||||
|
`mkdir -p cdc-versions/#{version}/lib`
|
||||||
|
`cp -r #{CDC_SOURCE_HOME}/flink-cdc-dist/src/main/flink-cdc-bin/* cdc-versions/#{version}/`
|
||||||
|
|
||||||
|
puts 'Compiling snapshot version...'
|
||||||
|
puts `cd #{CDC_SOURCE_HOME} && mvn clean install -DskipTests`
|
||||||
|
|
||||||
|
FILES.each do |lib|
|
||||||
|
if lib == 'dist'
|
||||||
|
`cp #{CDC_SOURCE_HOME}/flink-cdc-#{lib}/target/flink-cdc-#{lib}-#{version}.jar cdc-versions/#{version}/lib/`
|
||||||
|
else
|
||||||
|
`cp #{CDC_SOURCE_HOME}/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-#{lib}/target/flink-cdc-#{lib}-#{version}.jar cdc-versions/#{version}/lib/`
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
download_released
|
||||||
|
compile_snapshot HEAD_VERSION
|
||||||
|
|
||||||
|
puts 'Done'
|
@ -0,0 +1,162 @@
|
|||||||
|
#!/usr/bin/env ruby
|
||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
require 'pathname'
|
||||||
|
require 'securerandom'
|
||||||
|
|
||||||
|
WAITING_SECONDS = 20
|
||||||
|
FLINK_HOME = ENV['FLINK_HOME']
|
||||||
|
throw 'Unspecified `FLINK_HOME` environment variable.' if FLINK_HOME.nil?
|
||||||
|
FLINK_HOME = Pathname.new(FLINK_HOME).realpath
|
||||||
|
|
||||||
|
SOURCE_PORT = 3306
|
||||||
|
DATABASE_NAME = 'fallen'
|
||||||
|
TABLES = ['girl'].freeze
|
||||||
|
|
||||||
|
def exec_sql_source(sql)
|
||||||
|
`mysql -h 127.0.0.1 -P#{SOURCE_PORT} -uroot --skip-password -e "USE #{DATABASE_NAME}; #{sql}"`
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_mystery_data(mystery)
|
||||||
|
exec_sql_source("REPLACE INTO girl(id, name) VALUES (17, '#{mystery}');")
|
||||||
|
end
|
||||||
|
|
||||||
|
def ensure_mystery_data(mystery)
|
||||||
|
throw StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery
|
||||||
|
end
|
||||||
|
|
||||||
|
puts ' Waiting for source to start up...'
|
||||||
|
next until exec_sql_source("SELECT '1';") == "1\n1\n"
|
||||||
|
|
||||||
|
def test_migration_chore(from_version, to_version)
|
||||||
|
TABLES.each do |table_name|
|
||||||
|
exec_sql_source("DROP TABLE IF EXISTS #{table_name};")
|
||||||
|
exec_sql_source("CREATE TABLE #{table_name} (ID INT NOT NULL, NAME VARCHAR(17), PRIMARY KEY (ID));")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Clear previous savepoints and logs
|
||||||
|
`rm -rf savepoints`
|
||||||
|
|
||||||
|
# Prepare for current YAML file
|
||||||
|
test_route = !%w[3.0.0 3.0.1].include?(from_version)
|
||||||
|
yaml_job_template_file = test_route ? 'conf/pipeline-route.yaml' : 'conf/pipeline.yaml'
|
||||||
|
yaml_job_file = 'conf/temp.yaml'
|
||||||
|
yaml_content = File.open(yaml_job_template_file).read.gsub('${PIPELINE_NAME}',
|
||||||
|
"Pipeline Migration Job (#{from_version} -> #{to_version})")
|
||||||
|
File.write(yaml_job_file, yaml_content)
|
||||||
|
|
||||||
|
# Submit current pipeline job file
|
||||||
|
submit_job_output = `bash ./cdc-versions/#{from_version}/bin/flink-cdc.sh --flink-home #{FLINK_HOME} #{yaml_job_file}`
|
||||||
|
puts " #{submit_job_output}"
|
||||||
|
current_job_id = submit_job_output.split("\n")[1].split.last
|
||||||
|
raise StandardError, 'Failed to submit Flink job' unless current_job_id.length == 32
|
||||||
|
|
||||||
|
puts " Current Job ID: #{current_job_id}"
|
||||||
|
|
||||||
|
# Verify if data sync works
|
||||||
|
random_string_1 = SecureRandom.hex(8)
|
||||||
|
put_mystery_data random_string_1
|
||||||
|
sleep WAITING_SECONDS
|
||||||
|
ensure_mystery_data random_string_1
|
||||||
|
|
||||||
|
# Stop current job and create a savepoint
|
||||||
|
puts `#{FLINK_HOME}/bin/flink stop --savepointPath #{Dir.pwd}/savepoints #{current_job_id}`
|
||||||
|
savepoint_file = `ls savepoints`.split("\n").last
|
||||||
|
|
||||||
|
# Migrate to a newer CDC version
|
||||||
|
puts " Submitting CDC jobs at #{to_version}..."
|
||||||
|
submit_job_output = `bash ./cdc-versions/#{to_version}/bin/flink-cdc.sh --from-savepoint #{Dir.pwd}/savepoints/#{savepoint_file} --allow-nonRestored-state --flink-home #{FLINK_HOME} #{yaml_job_file}`
|
||||||
|
puts " #{submit_job_output}"
|
||||||
|
new_job_id = submit_job_output.split("\n")[1].split.last
|
||||||
|
raise StandardError, 'Failed to submit Flink job' unless new_job_id.length == 32
|
||||||
|
|
||||||
|
puts " Upgraded Job ID: #{new_job_id}"
|
||||||
|
|
||||||
|
# Verify if data sync works
|
||||||
|
puts "Submitted job at #{to_version} as #{new_job_id}"
|
||||||
|
random_string_2 = SecureRandom.hex(8)
|
||||||
|
put_mystery_data random_string_2
|
||||||
|
sleep WAITING_SECONDS
|
||||||
|
ensure_mystery_data random_string_2
|
||||||
|
puts `#{FLINK_HOME}/bin/flink cancel #{new_job_id}`
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_migration(from_version, to_version)
|
||||||
|
puts "➡️ [MIGRATION] Testing migration from #{from_version} to #{to_version}..."
|
||||||
|
puts " with Flink #{FLINK_HOME}..."
|
||||||
|
begin
|
||||||
|
result = test_migration_chore from_version, to_version
|
||||||
|
if result
|
||||||
|
puts "✅ [MIGRATION] Successfully migrated from #{from_version} to #{to_version}!"
|
||||||
|
else
|
||||||
|
puts "❌ [MIGRATION] Failed to migrate from #{from_version} to #{to_version}..."
|
||||||
|
end
|
||||||
|
result
|
||||||
|
rescue => e
|
||||||
|
puts "❌ [MIGRATION] Failed to migrate from #{from_version} to #{to_version}...", e
|
||||||
|
false
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
version_list = %w[3.0.0 3.0.1 3.1.0 3.1.1 3.2-SNAPSHOT]
|
||||||
|
no_savepoint_versions = %w[3.0.0 3.0.1]
|
||||||
|
version_result = Hash.new('❓')
|
||||||
|
@failures = []
|
||||||
|
|
||||||
|
version_list.each_with_index do |old_version, old_index|
|
||||||
|
puts 'Restarting cluster...'
|
||||||
|
`#{FLINK_HOME}/bin/stop-cluster.sh`
|
||||||
|
puts 'Stopped cluster.'
|
||||||
|
`#{FLINK_HOME}/bin/start-cluster.sh`
|
||||||
|
puts 'Started cluster.'
|
||||||
|
version_list.each_with_index do |new_version, new_index|
|
||||||
|
next if old_index > new_index
|
||||||
|
next if no_savepoint_versions.include? new_version
|
||||||
|
|
||||||
|
result = test_migration old_version, new_version
|
||||||
|
version_result[old_version + new_version] = result ? '✅' : '❌'
|
||||||
|
@failures << [old_version, new_version] unless result
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
printable_result = []
|
||||||
|
printable_result << [''] + version_list
|
||||||
|
version_list.each_with_index do |old_version, old_index|
|
||||||
|
table_line = [old_version]
|
||||||
|
version_list.each_with_index do |new_version, new_index|
|
||||||
|
table_line << if old_index > new_index
|
||||||
|
''
|
||||||
|
else
|
||||||
|
version_result[old_version + new_version]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
printable_result << table_line
|
||||||
|
end
|
||||||
|
|
||||||
|
begin
|
||||||
|
require 'terminal-table'
|
||||||
|
puts Terminal::Table.new rows: printable_result, title: 'Migration Test Result'
|
||||||
|
rescue LoadError
|
||||||
|
puts 'Test summary: ', printable_result
|
||||||
|
end
|
||||||
|
puts "✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`"
|
||||||
|
|
||||||
|
if @failures.filter { |old_version, new_version| new_version == version_list.last && old_version != '3.1.0' }.any?
|
||||||
|
abort 'Some migration to snapshot version tests failed.'
|
||||||
|
end
|
Loading…
Reference in New Issue