diff --git a/Dockerfile b/Dockerfile index ea5da8698..7653e551b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ FROM flink -ARG FLINK_CDC_VERSION=3.3-SNAPSHOT +ARG FLINK_CDC_VERSION=3.4-SNAPSHOT ARG PIPELINE_DEFINITION_FILE RUN mkdir -p /opt/flink-cdc diff --git a/docs/config.toml b/docs/config.toml index e3f39d30d..f79844e63 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -58,8 +58,9 @@ pygmentsUseClasses = true ] PreviousDocs = [ + ["3.3", "https://nightlies.apache.org/flink/flink-cdc-docs-release-3.3"], ["3.2", "https://nightlies.apache.org/flink/flink-cdc-docs-release-3.2"], - ["3.1", "https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1"], + ["3.1", "https://nightlies.apache.org/flink/flink-cdc-docs-release-3.1"] ] [markup] diff --git a/docs/content.zh/docs/connectors/pipeline-connectors/doris.md b/docs/content.zh/docs/connectors/pipeline-connectors/doris.md index 307abc138..4c442d814 100644 --- a/docs/content.zh/docs/connectors/pipeline-connectors/doris.md +++ b/docs/content.zh/docs/connectors/pipeline-connectors/doris.md @@ -169,7 +169,7 @@ pipeline: String StreamLoad的参数。 For example: sink.properties.strict_mode: true. - 查看更多关于 StreamLoad 的属性 + 查看更多关于 StreamLoad 的属性 @@ -179,7 +179,7 @@ pipeline: String 创建表的Properties配置。 For example: table.create.properties.replication_num: 1. - 查看更多关于 Doris Table 的属性 + 查看更多关于 Doris Table 的属性 diff --git a/docs/content/docs/connectors/pipeline-connectors/doris.md b/docs/content/docs/connectors/pipeline-connectors/doris.md index cee412c16..ffa769da4 100644 --- a/docs/content/docs/connectors/pipeline-connectors/doris.md +++ b/docs/content/docs/connectors/pipeline-connectors/doris.md @@ -169,7 +169,7 @@ pipeline: String Parameters of StreamLoad. For example: sink.properties.strict_mode: true. - See more about StreamLoad Properties + See more about StreamLoad Properties @@ -179,7 +179,7 @@ pipeline: String Create the Properties configuration of the table. For example: table.create.properties.replication_num: 1. - See more about Doris Table Properties + See more about Doris Table Properties diff --git a/pom.xml b/pom.xml index a3b6b0c4b..20c6bcdef 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ limitations under the License. - 3.3-SNAPSHOT + 3.4-SNAPSHOT 2.12 UTF-8 diff --git a/tools/mig-test/datastream/compile_jobs.rb b/tools/mig-test/datastream/compile_jobs.rb index d944ac91f..5c906e5bf 100644 --- a/tools/mig-test/datastream/compile_jobs.rb +++ b/tools/mig-test/datastream/compile_jobs.rb @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -JOB_VERSIONS = %w[3.2.0 3.2.1 3.3-SNAPSHOT] +JOB_VERSIONS = %w[3.2.0 3.2.1 3.3.0 3.4-SNAPSHOT] JOB_VERSIONS.each do |version| puts "Compiling DataStream job for CDC #{version}" diff --git a/tools/mig-test/datastream/datastream-3.3-SNAPSHOT/.gitignore b/tools/mig-test/datastream/datastream-3.3.0/.gitignore similarity index 100% rename from tools/mig-test/datastream/datastream-3.3-SNAPSHOT/.gitignore rename to tools/mig-test/datastream/datastream-3.3.0/.gitignore diff --git a/tools/mig-test/datastream/datastream-3.3-SNAPSHOT/pom.xml b/tools/mig-test/datastream/datastream-3.3.0/pom.xml similarity index 98% rename from tools/mig-test/datastream/datastream-3.3-SNAPSHOT/pom.xml rename to tools/mig-test/datastream/datastream-3.3.0/pom.xml index 4774f4fb8..cc65c6c78 100644 --- a/tools/mig-test/datastream/datastream-3.3-SNAPSHOT/pom.xml +++ b/tools/mig-test/datastream/datastream-3.3.0/pom.xml @@ -22,13 +22,13 @@ limitations under the License. org.apache.flink datastream-job - 3.3-SNAPSHOT + 3.3.0 jar UTF-8 1.19.1 - 3.3-SNAPSHOT + 3.3.0 1.9.7.Final 2.12 2.0.13 diff --git a/tools/mig-test/datastream/datastream-3.3-SNAPSHOT/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java similarity index 100% rename from tools/mig-test/datastream/datastream-3.3-SNAPSHOT/src/main/java/DataStreamJob.java rename to tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java diff --git a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/.gitignore b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/.gitignore new file mode 100644 index 000000000..5ff6309b7 --- /dev/null +++ b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml new file mode 100644 index 000000000..e174d5583 --- /dev/null +++ b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml @@ -0,0 +1,180 @@ + + + + 4.0.0 + + org.apache.flink + datastream-job + 3.4-SNAPSHOT + jar + + + UTF-8 + 1.19.1 + 3.4-SNAPSHOT + 1.9.7.Final + 2.12 + 2.0.13 + UTF-8 + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + + + org.apache.flink + flink-runtime + ${flink.version} + + + org.apache.flink + flink-java + ${flink.version} + + + org.apache.flink + flink-connector-base + ${flink.version} + + + org.apache.flink + flink-clients + ${flink.version} + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${flink.version} + + + org.apache.flink + flink-table-runtime + ${flink.version} + + + org.apache.flink + flink-core + ${flink.version} + + + org.apache.flink + flink-table-common + ${flink.version} + + + + + + + + + + org.apache.flink + flink-shaded-guava + 31.1-jre-17.0 + + + org.apache.flink + flink-connector-debezium + ${flink.cdc.version} + + + org.apache.flink + flink-cdc-base + ${flink.cdc.version} + + + org.apache.flink + flink-connector-mysql-cdc + ${flink.cdc.version} + + + io.debezium + debezium-connector-mysql + ${debezium.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-simple + ${slf4j.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + + maven-assembly-plugin + + + package + + single + + + + + + jar-with-dependencies + + + + + + + + java-8-target + + [1.8,11) + + + 1.8 + ${java.version} + ${java.version} + + + + java-11-target + + [11,) + + + 11 + ${java.version} + ${java.version} + + + + \ No newline at end of file diff --git a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java new file mode 100644 index 000000000..f821ac0a2 --- /dev/null +++ b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.flink.cdc.connectors.mysql.source.MySqlSource; +import org.apache.flink.cdc.connectors.mysql.table.StartupOptions; +import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +public class DataStreamJob { + + public static void main(String[] args) { + MySqlSource mySqlSource = MySqlSource.builder() + .hostname("localhost") + .port(3306) + .databaseList("fallen") + .tableList("fallen.angel", "fallen.gabriel", "fallen.girl") + .startupOptions(StartupOptions.initial()) + .username("root") + .password("") + .deserializer(new JsonDebeziumDeserializationSchema()) + .serverTimeZone("UTC") + .build(); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.enableCheckpointing(3000); + + env.fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL CDC Source") + .uid("sql-source-uid") + .setParallelism(1) + .print() + .setParallelism(1); + + try { + env.execute(); + } catch (Exception e) { + // ... unfortunately + } + } +} diff --git a/tools/mig-test/datastream/run_migration_test.rb b/tools/mig-test/datastream/run_migration_test.rb index e0cd8a589..deb16b0f0 100644 --- a/tools/mig-test/datastream/run_migration_test.rb +++ b/tools/mig-test/datastream/run_migration_test.rb @@ -93,7 +93,7 @@ def test_migration(from_version, to_version) end end -version_list = %w[3.2.0 3.2.1 3.3-SNAPSHOT] +version_list = %w[3.2.0 3.2.1 3.3.0 3.4-SNAPSHOT] version_result = Hash.new('❓') @failures = [] diff --git a/tools/mig-test/prepare_libs.rb b/tools/mig-test/prepare_libs.rb index 92175b7f9..4aceac8d0 100644 --- a/tools/mig-test/prepare_libs.rb +++ b/tools/mig-test/prepare_libs.rb @@ -38,9 +38,10 @@ end RELEASED_VERSIONS = { '3.2.0': gen_version('3.2.0'), '3.2.1': gen_version('3.2.1'), + '3.3.0': gen_version('3.3.0'), }.freeze -HEAD_VERSION = '3.3-SNAPSHOT' +HEAD_VERSION = '3.4-SNAPSHOT' def download_or_get(link) `mkdir -p cache` diff --git a/tools/mig-test/run_migration_test.rb b/tools/mig-test/run_migration_test.rb index d9f8d908e..bb111252c 100644 --- a/tools/mig-test/run_migration_test.rb +++ b/tools/mig-test/run_migration_test.rb @@ -118,8 +118,8 @@ def test_migration(from_version, to_version) end version_list = case ARGV[0] - when '1.19.1' then %w[3.2.0 3.2.1 3.3-SNAPSHOT] - when '1.20.0' then %w[3.2.1 3.3-SNAPSHOT] + when '1.19.1' then %w[3.2.0 3.2.1 3.3.0 3.4-SNAPSHOT] + when '1.20.0' then %w[3.2.1 3.3.0 3.4-SNAPSHOT] else [] end