gongzhongqiang
11 months ago
committed by
GitHub
parent
6da9684999
commit
f7fa317580
@ -0,0 +1,217 @@
|
||||
#
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Flink CDC CI
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- release-*
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- release-*
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
MODULES_CORE: "\
|
||||
flink-cdc-cli,\
|
||||
flink-cdc-common,\
|
||||
flink-cdc-composer,\
|
||||
flink-cdc-runtime"
|
||||
|
||||
MODULES_PIPELINE_CONNECTORS: "\
|
||||
flink-cdc-connect/flink-cdc-pipeline-connectors"
|
||||
|
||||
MODULES_MYSQL: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
|
||||
|
||||
MODULES_POSTGRES: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
|
||||
|
||||
MODULES_ORACLE: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
|
||||
|
||||
MODULES_MONGODB: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"
|
||||
|
||||
MODULES_SQLSERVER: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"
|
||||
|
||||
MODULES_TIDB: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
|
||||
|
||||
MODULES_OCEANBASE: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
|
||||
|
||||
MODULES_DB2: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
|
||||
|
||||
MODULES_VITESS: "\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
|
||||
|
||||
MODULES_E2E: "\
|
||||
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests,\
|
||||
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
|
||||
|
||||
jobs:
|
||||
compile_and_test:
|
||||
# Only run the CI pipeline for the flink-cdc-connectors repository
|
||||
# if: github.repository == 'apache/flink-cdc-connectors'
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
jdk: [ 8 ]
|
||||
module: [ "core",
|
||||
"pipeline_connectors",
|
||||
"mysql",
|
||||
"postgres",
|
||||
"oracle",
|
||||
"mongodb",
|
||||
"sqlserver",
|
||||
"tidb",
|
||||
"oceanbase",
|
||||
"db2",
|
||||
"vitess",
|
||||
"e2e"
|
||||
]
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
|
||||
-Dorg.slf4j.simpleLogger.showDateTime=true \
|
||||
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
|
||||
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
|
||||
--no-snapshot-updates -B \
|
||||
--settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
|
||||
-Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
|
||||
-Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
|
||||
steps:
|
||||
- run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"
|
||||
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set JDK
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: ${{ matrix.jdk }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
|
||||
- name: Set Maven 3.8.6
|
||||
uses: stCarolas/setup-maven@v4.5
|
||||
with:
|
||||
maven-version: 3.8.6
|
||||
|
||||
- name: Compile and test ${{ matrix.module }}
|
||||
timeout-minutes: 90
|
||||
run: |
|
||||
set -o pipefail
|
||||
|
||||
case ${{ matrix.module }} in
|
||||
("core")
|
||||
modules=${{ env.MODULES_CORE }}
|
||||
;;
|
||||
("pipeline_connectors")
|
||||
modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
|
||||
;;
|
||||
("mysql")
|
||||
modules=${{ env.MODULES_MYSQL }}
|
||||
;;
|
||||
("postgres")
|
||||
modules=${{ env.MODULES_POSTGRES }}
|
||||
;;
|
||||
("oracle")
|
||||
modules=${{ env.MODULES_ORACLE }}
|
||||
;;
|
||||
("mongodb")
|
||||
modules=${{ env.MODULES_MONGODB }}
|
||||
;;
|
||||
("sqlserver")
|
||||
modules=${{ env.MODULES_SQLSERVER }}
|
||||
;;
|
||||
("tidb")
|
||||
modules=${{ env.MODULES_TIDB }}
|
||||
;;
|
||||
("oceanbase")
|
||||
modules=${{ env.MODULES_OCEANBASE }}
|
||||
;;
|
||||
("db2")
|
||||
modules=${{ env.MODULES_DB2 }}
|
||||
;;
|
||||
("vitess")
|
||||
modules=${{ env.MODULES_VITESS }}
|
||||
;;
|
||||
("e2e")
|
||||
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_E2E }}"
|
||||
modules=${{ env.MODULES_E2E }}
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ ${{ matrix.module }} != "e2e" ]; then
|
||||
compile_modules=$modules
|
||||
fi
|
||||
|
||||
mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify
|
||||
|
||||
- name: Print JVM thread dumps when cancelled
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
# ----------------------------------------------------------------------------
|
||||
# Copyright 2023 The Netty Project
|
||||
#
|
||||
# ----------------------------------------------------------------------------
|
||||
# Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
|
||||
echo "$OSTYPE"
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
|
||||
echo "Setting up JVM thread dumps"
|
||||
# use jattach so that Java processes in docker containers are also covered
|
||||
# download jattach
|
||||
curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
|
||||
if command -v sha256sum &> /dev/null; then
|
||||
# verify hash of jattach binary
|
||||
sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
|
||||
fi
|
||||
chmod +x /tmp/jattach
|
||||
for java_pid in $(sudo pgrep java); do
|
||||
echo "----------------------- pid $java_pid -----------------------"
|
||||
echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
|
||||
sudo /tmp/jattach $java_pid jcmd VM.command_line || true
|
||||
sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
|
||||
sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
|
||||
done
|
||||
else
|
||||
for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
|
||||
echo "----------------------- pid $java_pid -----------------------"
|
||||
jcmd $java_pid VM.command_line || true
|
||||
jcmd $java_pid Thread.print -l
|
||||
jcmd $java_pid GC.heap_info || true
|
||||
done
|
||||
fi
|
||||
exit 0
|
||||
@ -1,47 +0,0 @@
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Maven
|
||||
# Build your Java project and run tests with Apache Maven.
|
||||
# Add steps that analyze code, save build artifacts, deploy, and more:
|
||||
# https://docs.microsoft.com/azure/devops/pipelines/languages/java
|
||||
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- '*' # must quote since "*" is a YAML reserved character; we want a string
|
||||
# Define variables:
|
||||
# - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
|
||||
# - See https://stackoverflow.com/questions/60742105/how-can-i-access-a-secret-value-from-an-azure-pipelines-expression
|
||||
# to understand why the secrets are handled like this
|
||||
variables:
|
||||
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
|
||||
E2E_CACHE_FOLDER: $(Pipeline.Workspace)/e2e_cache
|
||||
E2E_TARBALL_CACHE: $(Pipeline.Workspace)/e2e_artifact_cache
|
||||
MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
|
||||
CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
|
||||
CACHE_FALLBACK_KEY: maven | $(Agent.OS)
|
||||
FLINK_ARTIFACT_DIR: $(Pipeline.Workspace)/flink_artifact
|
||||
|
||||
stages:
|
||||
# CI / PR triggered stage:
|
||||
- stage: ci
|
||||
displayName: "CI build (custom builders)"
|
||||
jobs:
|
||||
- template: tools/azure-pipelines/jobs-template.yml
|
||||
parameters: # see template file for a definition of the parameters.
|
||||
stage_name: ci_build
|
||||
test_pool_definition:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
run_end_to_end: false
|
||||
jdk: 8
|
||||
@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
echo "Creating build artifact dir $FLINK_ARTIFACT_DIR"
|
||||
|
||||
cp -r . "$FLINK_ARTIFACT_DIR"
|
||||
|
||||
echo "Minimizing artifact files"
|
||||
|
||||
# reduces the size of the artifact directory to speed up
|
||||
# the packing&upload / download&unpacking process
|
||||
# by removing files not required for subsequent stages
|
||||
|
||||
# jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
|
||||
find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf
|
||||
|
||||
# .git directory
|
||||
# not deleting this can cause build stability issues
|
||||
# merging the cached version sometimes fails
|
||||
rm -rf "$FLINK_ARTIFACT_DIR/.git"
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
function prepare_debug_files {
|
||||
MODULE=$@
|
||||
export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files"
|
||||
export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r' '_')-$(date +%s)"
|
||||
echo "##vso[task.setvariable variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR"
|
||||
echo "##vso[task.setvariable variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME"
|
||||
mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; }
|
||||
}
|
||||
@ -1,197 +0,0 @@
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
parameters:
|
||||
test_pool_definition: # defines the hardware pool for compilation and unit test execution.
|
||||
stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage)
|
||||
run_end_to_end: # if set to 'true', the end to end tests will be executed
|
||||
jdk: # the jdk version to use
|
||||
|
||||
jobs:
|
||||
- job: compile_${{parameters.stage_name}}
|
||||
# succeeded() is needed to allow job cancellation
|
||||
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
|
||||
pool: ${{parameters.test_pool_definition}}
|
||||
timeoutInMinutes: 40
|
||||
cancelTimeoutInMinutes: 1
|
||||
workspace:
|
||||
clean: all # this cleans the entire workspace directory before running a new job
|
||||
# It is necessary because the custom build machines are reused for tests.
|
||||
# See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
|
||||
|
||||
steps:
|
||||
# The cache task is persisting the .m2 directory between builds, so that
|
||||
# we do not have to re-download all dependencies from maven central for
|
||||
# each build. The hope is that downloading the cache is faster than
|
||||
# all dependencies individually.
|
||||
# In this configuration, we use a hash over all committed (not generated) .pom files
|
||||
# as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash
|
||||
# (usually because a pom file has changed), we'll fall back to a key without
|
||||
# the pom files (CACHE_FALLBACK_KEY).
|
||||
# Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: $(CACHE_KEY)
|
||||
restoreKeys: $(CACHE_FALLBACK_KEY)
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
continueOnError: true # continue the build even if the cache fails.
|
||||
displayName: Cache Maven local repo
|
||||
- script: |
|
||||
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
|
||||
echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
|
||||
displayName: "Set JDK"
|
||||
# Compile
|
||||
- script: |
|
||||
./tools/ci/compile.sh || exit $?
|
||||
./tools/azure-pipelines/create_build_artifact.sh
|
||||
displayName: Compile
|
||||
|
||||
# upload artifacts for next stage
|
||||
- task: PublishPipelineArtifact@1
|
||||
inputs:
|
||||
targetPath: $(FLINK_ARTIFACT_DIR)
|
||||
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
|
||||
|
||||
- job: test_1_${{parameters.stage_name}}
|
||||
dependsOn: compile_${{parameters.stage_name}}
|
||||
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
|
||||
pool: ${{parameters.test_pool_definition}}
|
||||
timeoutInMinutes: 70
|
||||
cancelTimeoutInMinutes: 1
|
||||
workspace:
|
||||
clean: all
|
||||
strategy:
|
||||
matrix:
|
||||
core:
|
||||
module: core
|
||||
pipeline_connectors:
|
||||
module: pipeline_connectors
|
||||
postgres:
|
||||
module: postgres
|
||||
oracle:
|
||||
module: oracle
|
||||
mongodb:
|
||||
module: mongodb
|
||||
sqlserver:
|
||||
module: sqlserver
|
||||
tidb:
|
||||
module: tidb
|
||||
db2:
|
||||
module: db2
|
||||
vitess:
|
||||
module: vitess
|
||||
misc:
|
||||
module: misc
|
||||
steps:
|
||||
# download artifact from compile stage
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
path: $(FLINK_ARTIFACT_DIR)
|
||||
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
|
||||
|
||||
- script: ./tools/azure-pipelines/unpack_build_artifact.sh
|
||||
displayName: "Unpack Build artifact"
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: $(CACHE_KEY)
|
||||
restoreKeys: $(CACHE_FALLBACK_KEY)
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
continueOnError: true # continue the build even if the cache fails.
|
||||
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
|
||||
displayName: Cache Maven local repo
|
||||
|
||||
- script: |
|
||||
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
|
||||
echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
|
||||
displayName: "Set JDK"
|
||||
|
||||
- script: sudo sysctl -w kernel.core_pattern=core.%p
|
||||
displayName: Set coredump pattern
|
||||
|
||||
# Test
|
||||
- script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
|
||||
displayName: Test - $(module)
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit'
|
||||
|
||||
# upload debug artifacts
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', ''))
|
||||
displayName: Upload Logs
|
||||
inputs:
|
||||
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
|
||||
artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
|
||||
|
||||
- job: test_2_${{parameters.stage_name}}
|
||||
dependsOn: compile_${{parameters.stage_name}}
|
||||
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
|
||||
pool: ${{parameters.test_pool_definition}}
|
||||
timeoutInMinutes: 90
|
||||
cancelTimeoutInMinutes: 1
|
||||
workspace:
|
||||
clean: all
|
||||
strategy:
|
||||
matrix:
|
||||
mysql:
|
||||
module: mysql
|
||||
oceanbase:
|
||||
module: oceanbase
|
||||
e2e:
|
||||
module: e2e
|
||||
steps:
|
||||
# download artifact from compile stage
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
path: $(FLINK_ARTIFACT_DIR)
|
||||
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
|
||||
|
||||
- script: ./tools/azure-pipelines/unpack_build_artifact.sh
|
||||
displayName: "Unpack Build artifact"
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: $(CACHE_KEY)
|
||||
restoreKeys: $(CACHE_FALLBACK_KEY)
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
continueOnError: true # continue the build even if the cache fails.
|
||||
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
|
||||
displayName: Cache Maven local repo
|
||||
|
||||
- script: |
|
||||
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
|
||||
echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
|
||||
displayName: "Set JDK"
|
||||
|
||||
- script: sudo sysctl -w kernel.core_pattern=core.%p
|
||||
displayName: Set coredump pattern
|
||||
|
||||
# Test
|
||||
- script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
|
||||
displayName: Test - $(module)
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit'
|
||||
|
||||
# upload debug artifacts
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', ''))
|
||||
displayName: Upload Logs
|
||||
inputs:
|
||||
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
|
||||
artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
|
||||
@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
|
||||
if ! [ -e $FLINK_ARTIFACT_DIR ]; then
|
||||
echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Merging cache"
|
||||
cp -RT "$FLINK_ARTIFACT_DIR" "."
|
||||
|
||||
echo "Adjusting timestamps"
|
||||
|
||||
# adjust timestamps to prevent recompilation
|
||||
find . -type f -name '*.java' | xargs touch
|
||||
find . -type f -name '*.class' | xargs touch
|
||||
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This file has the following tasks
|
||||
# a) It reads the e2e timeout from the configuration file
|
||||
# b) It prints a warning if the test has reached 80% of it's execution time
|
||||
# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts
|
||||
|
||||
COMMAND=$@
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${HERE}/../ci/controller_utils.sh"
|
||||
|
||||
source ./tools/azure-pipelines/debug_files_utils.sh
|
||||
prepare_debug_files "$AGENT_JOBNAME"
|
||||
export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
|
||||
mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; }
|
||||
sudo apt-get install -y moreutils
|
||||
|
||||
REAL_START_SECONDS=$(date +"%s")
|
||||
REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT minutes" +"%s")
|
||||
REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS))
|
||||
KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60))
|
||||
|
||||
echo "Running command '$COMMAND' with a timeout of $(($REAL_TIMEOUT_SECONDS / 60)) minutes."
|
||||
|
||||
MAIN_PID_FILE="/tmp/uploading_watchdog_main.pid"
|
||||
|
||||
function timeout_watchdog() {
|
||||
# 95%
|
||||
sleep $(($REAL_TIMEOUT_SECONDS * 95 / 100))
|
||||
echo "=========================================================================================="
|
||||
echo "=== WARNING: This task took already 95% of the available time budget of $((REAL_TIMEOUT_SECONDS / 60)) minutes ==="
|
||||
echo "=========================================================================================="
|
||||
print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.0"
|
||||
|
||||
# final stack trace and kill processes 1 min before timeout
|
||||
local secondsToKill=$(($REAL_END_SECONDS - $(date +"%s") - $KILL_SECONDS_BEFORE_TIMEOUT))
|
||||
if [[ $secondsToKill -lt 0 ]]; then
|
||||
secondsToKill=0
|
||||
fi
|
||||
sleep $(secondsToKill)
|
||||
print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.1"
|
||||
|
||||
echo "============================="
|
||||
echo "=== WARNING: Killing task ==="
|
||||
echo "============================="
|
||||
pkill -P $(<$MAIN_PID_FILE) # kill descendants
|
||||
kill $(<$MAIN_PID_FILE) # kill process itself
|
||||
|
||||
exit 42
|
||||
}
|
||||
|
||||
timeout_watchdog &
|
||||
WATCHDOG_PID=$!
|
||||
|
||||
# ts from moreutils prepends the time to each line
|
||||
( $COMMAND & PID=$! ; echo $PID >$MAIN_PID_FILE ; wait $PID ) | ts | tee $DEBUG_FILES_OUTPUT_DIR/watchdog
|
||||
TEST_EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
# successful execution, cleanup watchdog related things
|
||||
if [[ "$TEST_EXIT_CODE" == 0 ]]; then
|
||||
kill $WATCHDOG_PID
|
||||
rm $DEBUG_FILES_OUTPUT_DIR/watchdog
|
||||
rm -f $DEBUG_FILES_OUTPUT_DIR/jps-traces.*
|
||||
fi
|
||||
|
||||
# properly forward exit code
|
||||
exit $TEST_EXIT_CODE
|
||||
@ -1,63 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains tooling for compiling Flink
|
||||
#
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1 # fail
|
||||
fi
|
||||
CI_DIR="$HERE/../ci"
|
||||
MVN_CLEAN_COMPILE_OUT="/tmp/clean_compile.out"
|
||||
|
||||
# source required ci scripts
|
||||
source "${CI_DIR}/stage.sh"
|
||||
source "${CI_DIR}/maven-utils.sh"
|
||||
|
||||
echo "Maven version:"
|
||||
run_mvn -version
|
||||
|
||||
echo "=============================================================================="
|
||||
echo "Compiling Flink CDC"
|
||||
echo "=============================================================================="
|
||||
|
||||
EXIT_CODE=0
|
||||
|
||||
run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests | tee $MVN_CLEAN_COMPILE_OUT
|
||||
|
||||
EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
if [ $EXIT_CODE != 0 ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Compiling Flink CDC failed."
|
||||
echo "=============================================================================="
|
||||
|
||||
grep "0 Unknown Licenses" target/rat.txt > /dev/null
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "License header check failure detected. Printing first 50 lines for convenience:"
|
||||
head -n 50 target/rat.txt
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
|
||||
@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
print_system_info() {
|
||||
echo "CPU information"
|
||||
lscpu
|
||||
|
||||
echo "Memory information"
|
||||
cat /proc/meminfo
|
||||
|
||||
echo "Disk information"
|
||||
df -hH
|
||||
|
||||
echo "Running build as"
|
||||
whoami
|
||||
}
|
||||
|
||||
print_stacktraces () {
|
||||
echo "=============================================================================="
|
||||
echo "The following Java processes are running (JPS)"
|
||||
echo "=============================================================================="
|
||||
|
||||
JAVA_PROCESSES=`jps`
|
||||
echo "$JAVA_PROCESSES"
|
||||
|
||||
local pids=( $(echo "$JAVA_PROCESSES" | awk '{print $1}') )
|
||||
|
||||
for pid in "${pids[@]}"; do
|
||||
echo "=============================================================================="
|
||||
echo "Printing stack trace of Java process ${pid}"
|
||||
echo "=============================================================================="
|
||||
|
||||
jstack $pid
|
||||
done
|
||||
}
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
<!--
|
||||
Copyright 2023 Ververica Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
<settings>
|
||||
<mirrors>
|
||||
<mirror>
|
||||
<id>google-maven-central</id>
|
||||
<name>GCS Maven Central mirror</name>
|
||||
<url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
|
||||
<mirrorOf>central</mirrorOf>
|
||||
</mirror>
|
||||
</mirrors>
|
||||
</settings>
|
||||
@ -1,32 +0,0 @@
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
rootLogger.level=INFO
|
||||
rootLogger.appenderRef.out.ref=FileAppender
|
||||
# -----------------------------------------------------------------------------
|
||||
# Console (use 'console')
|
||||
# -----------------------------------------------------------------------------
|
||||
appender.console.name=ConsoleAppender
|
||||
appender.console.type=CONSOLE
|
||||
appender.console.layout.type=PatternLayout
|
||||
appender.console.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n
|
||||
# -----------------------------------------------------------------------------
|
||||
# File (use 'file')
|
||||
# -----------------------------------------------------------------------------
|
||||
appender.file.name=FileAppender
|
||||
appender.file.type=FILE
|
||||
appender.file.fileName=${sys:log.dir}/mvn-${sys:mvn.forkNumber}.log
|
||||
appender.file.layout.type=PatternLayout
|
||||
appender.file.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n
|
||||
@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Utility for invoking Maven in CI
|
||||
function run_mvn {
|
||||
MVN_CMD="mvn"
|
||||
if [[ "$M2_HOME" != "" ]]; then
|
||||
MVN_CMD="${M2_HOME}/bin/mvn"
|
||||
fi
|
||||
|
||||
ARGS=$@
|
||||
INVOCATION="$MVN_CMD $MVN_GLOBAL_OPTIONS $ARGS"
|
||||
if [[ "$MVN_RUN_VERBOSE" != "false" ]]; then
|
||||
echo "Invoking mvn with '$INVOCATION'"
|
||||
fi
|
||||
eval $INVOCATION
|
||||
}
|
||||
export -f run_mvn
|
||||
|
||||
function setup_maven {
|
||||
set -e # fail if there was an error setting up maven
|
||||
if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then
|
||||
wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip
|
||||
unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip"
|
||||
rm "apache-maven-${MAVEN_VERSION}-bin.zip"
|
||||
fi
|
||||
|
||||
export M2_HOME="${MAVEN_VERSIONED_DIR}"
|
||||
echo "##vso[task.setvariable variable=M2_HOME]$M2_HOME"
|
||||
|
||||
# just in case: clean up the .m2 home and remove invalid jar files
|
||||
if [ -d "${HOME}/.m2/repository/" ]; then
|
||||
find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \;
|
||||
fi
|
||||
|
||||
echo "Installed Maven ${MAVEN_VERSION} to ${M2_HOME}"
|
||||
set +e
|
||||
}
|
||||
|
||||
function set_mirror_config {
|
||||
if [[ "$MAVEN_MIRROR_CONFIG_FILE" != "" ]]; then
|
||||
echo "[WARN] Maven mirror already configured to $MAVEN_MIRROR_CONFIG_FILE"
|
||||
exit 0;
|
||||
fi
|
||||
|
||||
echo "Using Google mirror"
|
||||
MAVEN_MIRROR_CONFIG_FILE="$CI_DIR/google-mirror-settings.xml"
|
||||
}
|
||||
|
||||
function collect_coredumps {
|
||||
local SEARCHDIR=$1
|
||||
local TARGET_DIR=$2
|
||||
echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'"
|
||||
for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do
|
||||
echo "Moving '$file' to target directory ('$TARGET_DIR')"
|
||||
mv $file $TARGET_DIR/$(echo $file | tr "/" "-")
|
||||
done
|
||||
}
|
||||
|
||||
function collect_dmesg {
|
||||
local TARGET_DIR=$1
|
||||
dmesg > $TARGET_DIR/dmesg.out
|
||||
}
|
||||
|
||||
CI_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
MAVEN_VERSION="3.2.5"
|
||||
MAVEN_CACHE_DIR=${HOME}/maven_cache
|
||||
MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION}
|
||||
|
||||
|
||||
MAVEN_MIRROR_CONFIG_FILE=""
|
||||
set_mirror_config
|
||||
|
||||
export MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR=""
|
||||
# see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dmaven.wagon.http.pool=false "
|
||||
# logging
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn "
|
||||
# suppress snapshot updates
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="--no-snapshot-updates "
|
||||
# enable non-interactive batch mode
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-B "
|
||||
# globally control the build profile details
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="$PROFILE "
|
||||
|
||||
export MVN_GLOBAL_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} "
|
||||
# use google mirror everywhere
|
||||
MVN_GLOBAL_OPTIONS+="--settings $MAVEN_MIRROR_CONFIG_FILE "
|
||||
@ -1,198 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
STAGE_CORE="core"
|
||||
STAGE_PIPELINE_CONNECTORS="pipeline_connectors"
|
||||
STAGE_MYSQL="mysql"
|
||||
STAGE_POSTGRES="postgres"
|
||||
STAGE_ORACLE="oracle"
|
||||
STAGE_MONGODB="mongodb"
|
||||
STAGE_SQLSERVER="sqlserver"
|
||||
STAGE_TIDB="tidb"
|
||||
STAGE_OCEANBASE="oceanbase"
|
||||
STAGE_DB2="db2"
|
||||
STAGE_VITESS="vitess"
|
||||
STAGE_E2E="e2e"
|
||||
STAGE_MISC="misc"
|
||||
|
||||
MODULES_CORE="\
|
||||
flink-cdc-cli,\
|
||||
flink-cdc-common,\
|
||||
flink-cdc-composer,\
|
||||
flink-cdc-runtime"
|
||||
|
||||
MODULES_PIPELINE_CONNECTORS="\
|
||||
flink-cdc-connect/flink-cdc-pipeline-connectors"
|
||||
|
||||
MODULES_MYSQL="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
|
||||
|
||||
MODULES_POSTGRES="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
|
||||
|
||||
MODULES_ORACLE="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
|
||||
|
||||
MODULES_MONGODB="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"
|
||||
|
||||
MODULES_SQLSERVER="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"
|
||||
|
||||
MODULES_TIDB="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
|
||||
|
||||
MODULES_OCEANBASE="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
|
||||
|
||||
MODULES_DB2="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
|
||||
|
||||
MODULES_VITESS="\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
|
||||
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
|
||||
|
||||
MODULES_E2E="\
|
||||
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests,\
|
||||
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
|
||||
|
||||
function get_compile_modules_for_stage() {
|
||||
local stage=$1
|
||||
|
||||
case ${stage} in
|
||||
(${STAGE_CORE})
|
||||
echo "-pl $MODULES_CORE -am"
|
||||
;;
|
||||
(${STAGE_PIPELINE_CONNECTORS})
|
||||
echo "-pl $MODULES_PIPELINE_CONNECTORS -am"
|
||||
;;
|
||||
(${STAGE_MYSQL})
|
||||
echo "-pl $MODULES_MYSQL -am"
|
||||
;;
|
||||
(${STAGE_POSTGRES})
|
||||
echo "-pl $MODULES_POSTGRES -am"
|
||||
;;
|
||||
(${STAGE_ORACLE})
|
||||
echo "-pl $MODULES_ORACLE -am"
|
||||
;;
|
||||
(${STAGE_MONGODB})
|
||||
echo "-pl $MODULES_MONGODB -am"
|
||||
;;
|
||||
(${STAGE_SQLSERVER})
|
||||
echo "-pl $MODULES_SQLSERVER -am"
|
||||
;;
|
||||
(${STAGE_TIDB})
|
||||
echo "-pl $MODULES_TIDB -am"
|
||||
;;
|
||||
(${STAGE_OCEANBASE})
|
||||
echo "-pl $MODULES_OCEANBASE -am"
|
||||
;;
|
||||
(${STAGE_DB2})
|
||||
echo "-pl $MODULES_DB2 -am"
|
||||
;;
|
||||
(${STAGE_VITESS})
|
||||
echo "-pl $MODULES_VITESS -am"
|
||||
;;
|
||||
(${STAGE_E2E})
|
||||
# compile everything; using the -am switch does not work with negated module lists!
|
||||
# the negation takes precedence, thus not all required modules would be built
|
||||
echo ""
|
||||
;;
|
||||
(${STAGE_MISC})
|
||||
# compile everything; using the -am switch does not work with negated module lists!
|
||||
# the negation takes precedence, thus not all required modules would be built
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function get_test_modules_for_stage() {
|
||||
local stage=$1
|
||||
|
||||
local modules_core=$MODULES_CORE
|
||||
local modules_pipeline_connectors=$MODULES_PIPELINE_CONNECTORS
|
||||
local modules_mysql=$MODULES_MYSQL
|
||||
local modules_postgres=$MODULES_POSTGRES
|
||||
local modules_oracle=$MODULES_ORACLE
|
||||
local modules_mongodb=$MODULES_MONGODB
|
||||
local modules_sqlserver=$MODULES_SQLSERVER
|
||||
local modules_tidb=$MODULES_TIDB
|
||||
local modules_oceanbase=$MODULES_OCEANBASE
|
||||
local modules_db2=$MODULES_DB2
|
||||
local modules_vitess=$MODULES_VITESS
|
||||
local modules_e2e=$MODULES_E2E
|
||||
local negated_mysql=\!${MODULES_MYSQL//,/,\!}
|
||||
local negated_postgres=\!${MODULES_POSTGRES//,/,\!}
|
||||
local negated_oracle=\!${MODULES_ORACLE//,/,\!}
|
||||
local negated_mongodb=\!${MODULES_MONGODB//,/,\!}
|
||||
local negated_sqlserver=\!${MODULES_SQLSERVER//,/,\!}
|
||||
local negated_tidb=\!${MODULES_TIDB//,/,\!}
|
||||
local negated_oceanbase=\!${MODULES_OCEANBASE//,/,\!}
|
||||
local negated_db2=\!${MODULES_DB2//,/,\!}
|
||||
local negated_vitess=\!${MODULES_vitess//,/,\!}
|
||||
local negated_e2e=\!${MODULES_E2E//,/,\!}
|
||||
local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb,$negated_sqlserver,$negated_tidb,$negated_oceanbase,$negated_db2,$negated_vitess,$negated_e2e"
|
||||
|
||||
case ${stage} in
|
||||
(${STAGE_CORE})
|
||||
echo "-pl $modules_core"
|
||||
;;
|
||||
(${STAGE_PIPELINE_CONNECTORS})
|
||||
echo "-pl $modules_pipeline_connectors"
|
||||
;;
|
||||
(${STAGE_MYSQL})
|
||||
echo "-pl $modules_mysql"
|
||||
;;
|
||||
(${STAGE_POSTGRES})
|
||||
echo "-pl $modules_postgres"
|
||||
;;
|
||||
(${STAGE_ORACLE})
|
||||
echo "-pl $modules_oracle"
|
||||
;;
|
||||
(${STAGE_MONGODB})
|
||||
echo "-pl $modules_mongodb"
|
||||
;;
|
||||
(${STAGE_SQLSERVER})
|
||||
echo "-pl $modules_sqlserver"
|
||||
;;
|
||||
(${STAGE_TIDB})
|
||||
echo "-pl $modules_tidb"
|
||||
;;
|
||||
(${STAGE_OCEANBASE})
|
||||
echo "-pl $modules_oceanbase"
|
||||
;;
|
||||
(${STAGE_DB2})
|
||||
echo "-pl $modules_db2"
|
||||
;;
|
||||
(${STAGE_VITESS})
|
||||
echo "-pl $modules_vitess"
|
||||
;;
|
||||
(${STAGE_E2E})
|
||||
echo "-pl $modules_e2e"
|
||||
;;
|
||||
(${STAGE_MISC})
|
||||
echo "-pl $modules_misc"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains generic control over the test execution.
|
||||
#
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${HERE}/stage.sh"
|
||||
source "${HERE}/maven-utils.sh"
|
||||
source "${HERE}/controller_utils.sh"
|
||||
|
||||
STAGE=$1
|
||||
|
||||
# =============================================================================
|
||||
# Step 0: Check & print environment information & configure env
|
||||
# =============================================================================
|
||||
|
||||
# check preconditions
|
||||
if [ -z "$DEBUG_FILES_OUTPUT_DIR" ] ; then
|
||||
echo "ERROR: Environment variable 'DEBUG_FILES_OUTPUT_DIR' is not set but expected by test_controller.sh. Tests may use this location to store debugging files."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$DEBUG_FILES_OUTPUT_DIR" ] ; then
|
||||
echo "ERROR: Environment variable DEBUG_FILES_OUTPUT_DIR=$DEBUG_FILES_OUTPUT_DIR points to a directory that does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$STAGE" ] ; then
|
||||
echo "ERROR: Environment variable 'STAGE' is not set but expected by test_controller.sh. The variable refers to the stage being executed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Printing environment information"
|
||||
|
||||
echo "PATH=$PATH"
|
||||
run_mvn -version
|
||||
echo "Commit: $(git rev-parse HEAD)"
|
||||
print_system_info
|
||||
|
||||
# enable coredumps for this process
|
||||
ulimit -c unlimited
|
||||
|
||||
# configure JVMs to produce heap dumps
|
||||
export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
|
||||
|
||||
# some tests provide additional logs if they find this variable
|
||||
export IS_CI=true
|
||||
|
||||
export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log"
|
||||
|
||||
source "${HERE}/watchdog.sh"
|
||||
|
||||
# =============================================================================
|
||||
# Step 1: Rebuild jars and install Flink to local maven repository
|
||||
# =============================================================================
|
||||
|
||||
LOG4J_PROPERTIES=${HERE}/log4j.properties
|
||||
MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
|
||||
|
||||
MVN_COMMON_OPTIONS="-Dfast $MVN_LOGGING_OPTIONS"
|
||||
MVN_COMPILE_OPTIONS="-DskipTests"
|
||||
MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE})
|
||||
|
||||
CALLBACK_ON_TIMEOUT="print_stacktraces | tee ${DEBUG_FILES_OUTPUT_DIR}/jps-traces.out"
|
||||
run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install" $CALLBACK_ON_TIMEOUT
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE != 0 ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Compilation failure detected, skipping test execution."
|
||||
echo "=============================================================================="
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Step 2: Run tests
|
||||
# =============================================================================
|
||||
|
||||
MVN_TEST_MODULES=$(get_test_modules_for_stage ${STAGE})
|
||||
|
||||
run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $PROFILE $MVN_TEST_MODULES verify" $CALLBACK_ON_TIMEOUT
|
||||
EXIT_CODE=$?
|
||||
|
||||
# =============================================================================
|
||||
# Step 3: Put extra logs into $DEBUG_FILES_OUTPUT_DIR
|
||||
# =============================================================================
|
||||
|
||||
collect_coredumps $(pwd) $DEBUG_FILES_OUTPUT_DIR
|
||||
|
||||
# Exit code for CI build success/failure
|
||||
exit $EXIT_CODE
|
||||
@ -1,127 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Copyright 2023 Ververica Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains a watchdog tool to monitor a task and potentially kill it after
|
||||
# not producing any output for $MAX_NO_OUTPUT seconds.
|
||||
#
|
||||
|
||||
# Number of seconds w/o output before printing a stack trace and killing the watched process
|
||||
MAX_NO_OUTPUT=${MAX_NO_OUTPUT:-900}
|
||||
|
||||
# Number of seconds to sleep before checking the output again
|
||||
SLEEP_TIME=${SLEEP_TIME:-20}
|
||||
|
||||
# Internal fields
|
||||
CMD_OUT="/tmp/watchdog.out"
|
||||
CMD_PID="/tmp/watchdog.pid"
|
||||
CMD_EXIT="/tmp/watchdog.exit"
|
||||
|
||||
|
||||
# =============================================
|
||||
# Utility functions
|
||||
# =============================================
|
||||
|
||||
max_of() {
|
||||
local max number
|
||||
|
||||
max="$1"
|
||||
|
||||
for number in "${@:2}"; do
|
||||
if ((number > max)); then
|
||||
max="$number"
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%d\n' "$max"
|
||||
}
|
||||
|
||||
# Returns the highest modification time out of $CMD_OUT (which is the command output file)
|
||||
# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests)
|
||||
mod_time () {
|
||||
CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT`
|
||||
ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null`
|
||||
echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES`
|
||||
}
|
||||
|
||||
the_time() {
|
||||
echo `date +%s`
|
||||
}
|
||||
|
||||
# watchdog process
|
||||
|
||||
watchdog () {
|
||||
touch $CMD_OUT
|
||||
|
||||
while true; do
|
||||
sleep $SLEEP_TIME
|
||||
|
||||
time_diff=$((`the_time` - `mod_time`))
|
||||
|
||||
if [ $time_diff -ge $MAX_NO_OUTPUT ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Process produced no output for ${MAX_NO_OUTPUT} seconds."
|
||||
echo "=============================================================================="
|
||||
|
||||
# run timeout callback
|
||||
$CALLBACK_ON_TIMEOUT
|
||||
|
||||
echo "Killing process with pid=$(<$CMD_PID) and all descendants"
|
||||
pkill -P $(<$CMD_PID) # kill descendants
|
||||
kill $(<$CMD_PID) # kill process itself
|
||||
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# =============================================
|
||||
# main function
|
||||
# =============================================
|
||||
|
||||
# entrypoint
|
||||
function run_with_watchdog() {
|
||||
local cmd="$1"
|
||||
local CALLBACK_ON_TIMEOUT="$2"
|
||||
|
||||
watchdog &
|
||||
WD_PID=$!
|
||||
echo "STARTED watchdog (${WD_PID})."
|
||||
|
||||
echo "RUNNING '${cmd}'."
|
||||
|
||||
# Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to
|
||||
# allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains
|
||||
# the exit code. This is important for CI build life-cycle (success/failure).
|
||||
( $cmd & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT
|
||||
|
||||
EXIT_CODE=$(<$CMD_EXIT)
|
||||
|
||||
echo "Process exited with EXIT CODE: ${EXIT_CODE}."
|
||||
|
||||
# Make sure to kill the watchdog in any case after $CMD has completed
|
||||
echo "Trying to KILL watchdog (${WD_PID})."
|
||||
( kill $WD_PID 2>&1 ) > /dev/null
|
||||
|
||||
rm $CMD_PID
|
||||
rm $CMD_EXIT
|
||||
|
||||
return $EXIT_CODE
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue