diff --git a/.github/workflows/flink_cdc.yml b/.github/workflows/flink_cdc.yml new file mode 100644 index 000000000..9c977ebab --- /dev/null +++ b/.github/workflows/flink_cdc.yml @@ -0,0 +1,217 @@ +# +# Copyright 2023 Ververica Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Flink CDC CI +on: + push: + branches: + - master + - release-* + pull_request: + branches: + - master + - release-* + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MODULES_CORE: "\ + flink-cdc-cli,\ + flink-cdc-common,\ + flink-cdc-composer,\ + flink-cdc-runtime" + + MODULES_PIPELINE_CONNECTORS: "\ + flink-cdc-connect/flink-cdc-pipeline-connectors" + + MODULES_MYSQL: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc" + + MODULES_POSTGRES: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc" + + MODULES_ORACLE: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc" + + MODULES_MONGODB: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc" + + MODULES_SQLSERVER: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc" + + MODULES_TIDB: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc" + + MODULES_OCEANBASE: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc" + + MODULES_DB2: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc" + + MODULES_VITESS: "\ + flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\ + flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc" + + MODULES_E2E: "\ + flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests,\ + flink-cdc-e2e-tests/flink-cdc-source-e2e-tests" + +jobs: + compile_and_test: + # Only run the CI pipeline for the flink-cdc-connectors repository +# if: github.repository == 'apache/flink-cdc-connectors' + runs-on: ubuntu-latest + strategy: + matrix: + jdk: [ 8 ] + module: [ "core", + "pipeline_connectors", + "mysql", + "postgres", + "oracle", + "mongodb", + "sqlserver", + "tidb", + "oceanbase", + "db2", + "vitess", + "e2e" + ] + timeout-minutes: 120 + env: + MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + --no-snapshot-updates -B \ + --settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \ + -Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \ + -Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties + steps: + - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}" + + - name: Check out repository code + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set JDK + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' + + - name: Set Maven 3.8.6 + uses: stCarolas/setup-maven@v4.5 + with: + maven-version: 3.8.6 + + - name: Compile and test ${{ matrix.module }} + timeout-minutes: 90 + run: | + set -o pipefail + + case ${{ matrix.module }} in + ("core") + modules=${{ env.MODULES_CORE }} + ;; + ("pipeline_connectors") + modules=${{ env.MODULES_PIPELINE_CONNECTORS }} + ;; + ("mysql") + modules=${{ env.MODULES_MYSQL }} + ;; + ("postgres") + modules=${{ env.MODULES_POSTGRES }} + ;; + ("oracle") + modules=${{ env.MODULES_ORACLE }} + ;; + ("mongodb") + modules=${{ env.MODULES_MONGODB }} + ;; + ("sqlserver") + modules=${{ env.MODULES_SQLSERVER }} + ;; + ("tidb") + modules=${{ env.MODULES_TIDB }} + ;; + ("oceanbase") + modules=${{ env.MODULES_OCEANBASE }} + ;; + ("db2") + modules=${{ env.MODULES_DB2 }} + ;; + ("vitess") + modules=${{ env.MODULES_VITESS }} + ;; + ("e2e") + compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_E2E }}" + modules=${{ env.MODULES_E2E }} + ;; + esac + + if [ ${{ matrix.module }} != "e2e" ]; then + compile_modules=$modules + fi + + mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify + + - name: Print JVM thread dumps when cancelled + if: ${{ failure() }} + run: | + # ---------------------------------------------------------------------------- + # Copyright 2023 The Netty Project + # + # ---------------------------------------------------------------------------- + # Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml + echo "$OSTYPE" + if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then + echo "Setting up JVM thread dumps" + # use jattach so that Java processes in docker containers are also covered + # download jattach + curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach + if command -v sha256sum &> /dev/null; then + # verify hash of jattach binary + sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1 + fi + chmod +x /tmp/jattach + for java_pid in $(sudo pgrep java); do + echo "----------------------- pid $java_pid -----------------------" + echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)" + sudo /tmp/jattach $java_pid jcmd VM.command_line || true + sudo /tmp/jattach $java_pid jcmd "Thread.print -l" + sudo /tmp/jattach $java_pid jcmd GC.heap_info || true + done + else + for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do + echo "----------------------- pid $java_pid -----------------------" + jcmd $java_pid VM.command_line || true + jcmd $java_pid Thread.print -l + jcmd $java_pid GC.heap_info || true + done + fi + exit 0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 4785ce649..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Maven -# Build your Java project and run tests with Apache Maven. -# Add steps that analyze code, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/java - -trigger: - branches: - include: - - '*' # must quote since "*" is a YAML reserved character; we want a string -# Define variables: -# - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching -# - See https://stackoverflow.com/questions/60742105/how-can-i-access-a-secret-value-from-an-azure-pipelines-expression -# to understand why the secrets are handled like this -variables: - MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository - E2E_CACHE_FOLDER: $(Pipeline.Workspace)/e2e_cache - E2E_TARBALL_CACHE: $(Pipeline.Workspace)/e2e_artifact_cache - MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' - CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/** - CACHE_FALLBACK_KEY: maven | $(Agent.OS) - FLINK_ARTIFACT_DIR: $(Pipeline.Workspace)/flink_artifact - -stages: - # CI / PR triggered stage: - - stage: ci - displayName: "CI build (custom builders)" - jobs: - - template: tools/azure-pipelines/jobs-template.yml - parameters: # see template file for a definition of the parameters. - stage_name: ci_build - test_pool_definition: - vmImage: 'ubuntu-20.04' - run_end_to_end: false - jdk: 8 diff --git a/tools/azure-pipelines/create_build_artifact.sh b/tools/azure-pipelines/create_build_artifact.sh deleted file mode 100755 index ab34b50b3..000000000 --- a/tools/azure-pipelines/create_build_artifact.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -echo "Creating build artifact dir $FLINK_ARTIFACT_DIR" - -cp -r . "$FLINK_ARTIFACT_DIR" - -echo "Minimizing artifact files" - -# reduces the size of the artifact directory to speed up -# the packing&upload / download&unpacking process -# by removing files not required for subsequent stages - -# jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) -find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf - -# .git directory -# not deleting this can cause build stability issues -# merging the cached version sometimes fails -rm -rf "$FLINK_ARTIFACT_DIR/.git" - diff --git a/tools/azure-pipelines/debug_files_utils.sh b/tools/azure-pipelines/debug_files_utils.sh deleted file mode 100755 index 232c41afa..000000000 --- a/tools/azure-pipelines/debug_files_utils.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -function prepare_debug_files { - MODULE=$@ - export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files" - export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r' '_')-$(date +%s)" - echo "##vso[task.setvariable variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR" - echo "##vso[task.setvariable variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME" - mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; } -} diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml deleted file mode 100644 index 386862279..000000000 --- a/tools/azure-pipelines/jobs-template.yml +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -parameters: - test_pool_definition: # defines the hardware pool for compilation and unit test execution. - stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage) - run_end_to_end: # if set to 'true', the end to end tests will be executed - jdk: # the jdk version to use - -jobs: - - job: compile_${{parameters.stage_name}} - # succeeded() is needed to allow job cancellation - condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) - pool: ${{parameters.test_pool_definition}} - timeoutInMinutes: 40 - cancelTimeoutInMinutes: 1 - workspace: - clean: all # this cleans the entire workspace directory before running a new job - # It is necessary because the custom build machines are reused for tests. - # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace - - steps: - # The cache task is persisting the .m2 directory between builds, so that - # we do not have to re-download all dependencies from maven central for - # each build. The hope is that downloading the cache is faster than - # all dependencies individually. - # In this configuration, we use a hash over all committed (not generated) .pom files - # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash - # (usually because a pom file has changed), we'll fall back to a key without - # the pom files (CACHE_FALLBACK_KEY). - # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops - - task: Cache@2 - inputs: - key: $(CACHE_KEY) - restoreKeys: $(CACHE_FALLBACK_KEY) - path: $(MAVEN_CACHE_FOLDER) - continueOnError: true # continue the build even if the cache fails. - displayName: Cache Maven local repo - - script: | - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" - echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" - displayName: "Set JDK" - # Compile - - script: | - ./tools/ci/compile.sh || exit $? - ./tools/azure-pipelines/create_build_artifact.sh - displayName: Compile - - # upload artifacts for next stage - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(FLINK_ARTIFACT_DIR) - artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - - job: test_1_${{parameters.stage_name}} - dependsOn: compile_${{parameters.stage_name}} - condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) - pool: ${{parameters.test_pool_definition}} - timeoutInMinutes: 70 - cancelTimeoutInMinutes: 1 - workspace: - clean: all - strategy: - matrix: - core: - module: core - pipeline_connectors: - module: pipeline_connectors - postgres: - module: postgres - oracle: - module: oracle - mongodb: - module: mongodb - sqlserver: - module: sqlserver - tidb: - module: tidb - db2: - module: db2 - vitess: - module: vitess - misc: - module: misc - steps: - # download artifact from compile stage - - task: DownloadPipelineArtifact@2 - inputs: - path: $(FLINK_ARTIFACT_DIR) - artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - - script: ./tools/azure-pipelines/unpack_build_artifact.sh - displayName: "Unpack Build artifact" - - - task: Cache@2 - inputs: - key: $(CACHE_KEY) - restoreKeys: $(CACHE_FALLBACK_KEY) - path: $(MAVEN_CACHE_FOLDER) - continueOnError: true # continue the build even if the cache fails. - condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) - displayName: Cache Maven local repo - - - script: | - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" - echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" - displayName: "Set JDK" - - - script: sudo sysctl -w kernel.core_pattern=core.%p - displayName: Set coredump pattern - - # Test - - script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) - displayName: Test - $(module) - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFormat: 'JUnit' - - # upload debug artifacts - - task: PublishPipelineArtifact@1 - condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', '')) - displayName: Upload Logs - inputs: - targetPath: $(DEBUG_FILES_OUTPUT_DIR) - artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) - - - job: test_2_${{parameters.stage_name}} - dependsOn: compile_${{parameters.stage_name}} - condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) - pool: ${{parameters.test_pool_definition}} - timeoutInMinutes: 90 - cancelTimeoutInMinutes: 1 - workspace: - clean: all - strategy: - matrix: - mysql: - module: mysql - oceanbase: - module: oceanbase - e2e: - module: e2e - steps: - # download artifact from compile stage - - task: DownloadPipelineArtifact@2 - inputs: - path: $(FLINK_ARTIFACT_DIR) - artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - - script: ./tools/azure-pipelines/unpack_build_artifact.sh - displayName: "Unpack Build artifact" - - - task: Cache@2 - inputs: - key: $(CACHE_KEY) - restoreKeys: $(CACHE_FALLBACK_KEY) - path: $(MAVEN_CACHE_FOLDER) - continueOnError: true # continue the build even if the cache fails. - condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) - displayName: Cache Maven local repo - - - script: | - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" - echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" - displayName: "Set JDK" - - - script: sudo sysctl -w kernel.core_pattern=core.%p - displayName: Set coredump pattern - - # Test - - script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) - displayName: Test - $(module) - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFormat: 'JUnit' - - # upload debug artifacts - - task: PublishPipelineArtifact@1 - condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', '')) - displayName: Upload Logs - inputs: - targetPath: $(DEBUG_FILES_OUTPUT_DIR) - artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) \ No newline at end of file diff --git a/tools/azure-pipelines/unpack_build_artifact.sh b/tools/azure-pipelines/unpack_build_artifact.sh deleted file mode 100755 index 367c07c97..000000000 --- a/tools/azure-pipelines/unpack_build_artifact.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - - -if ! [ -e $FLINK_ARTIFACT_DIR ]; then - echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build." - exit 1 -fi - -echo "Merging cache" -cp -RT "$FLINK_ARTIFACT_DIR" "." - -echo "Adjusting timestamps" - -# adjust timestamps to prevent recompilation -find . -type f -name '*.java' | xargs touch -find . -type f -name '*.class' | xargs touch diff --git a/tools/azure-pipelines/uploading_watchdog.sh b/tools/azure-pipelines/uploading_watchdog.sh deleted file mode 100755 index 50156f391..000000000 --- a/tools/azure-pipelines/uploading_watchdog.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file has the following tasks -# a) It reads the e2e timeout from the configuration file -# b) It prints a warning if the test has reached 80% of it's execution time -# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts - -COMMAND=$@ - -HERE="`dirname \"$0\"`" # relative -HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized -if [ -z "$HERE" ] ; then - exit 1 -fi - -source "${HERE}/../ci/controller_utils.sh" - -source ./tools/azure-pipelines/debug_files_utils.sh -prepare_debug_files "$AGENT_JOBNAME" -export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs" -mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; } -sudo apt-get install -y moreutils - -REAL_START_SECONDS=$(date +"%s") -REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT minutes" +"%s") -REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS)) -KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60)) - -echo "Running command '$COMMAND' with a timeout of $(($REAL_TIMEOUT_SECONDS / 60)) minutes." - -MAIN_PID_FILE="/tmp/uploading_watchdog_main.pid" - -function timeout_watchdog() { - # 95% - sleep $(($REAL_TIMEOUT_SECONDS * 95 / 100)) - echo "==========================================================================================" - echo "=== WARNING: This task took already 95% of the available time budget of $((REAL_TIMEOUT_SECONDS / 60)) minutes ===" - echo "==========================================================================================" - print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.0" - - # final stack trace and kill processes 1 min before timeout - local secondsToKill=$(($REAL_END_SECONDS - $(date +"%s") - $KILL_SECONDS_BEFORE_TIMEOUT)) - if [[ $secondsToKill -lt 0 ]]; then - secondsToKill=0 - fi - sleep $(secondsToKill) - print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.1" - - echo "=============================" - echo "=== WARNING: Killing task ===" - echo "=============================" - pkill -P $(<$MAIN_PID_FILE) # kill descendants - kill $(<$MAIN_PID_FILE) # kill process itself - - exit 42 -} - -timeout_watchdog & -WATCHDOG_PID=$! - -# ts from moreutils prepends the time to each line -( $COMMAND & PID=$! ; echo $PID >$MAIN_PID_FILE ; wait $PID ) | ts | tee $DEBUG_FILES_OUTPUT_DIR/watchdog -TEST_EXIT_CODE=${PIPESTATUS[0]} - -# successful execution, cleanup watchdog related things -if [[ "$TEST_EXIT_CODE" == 0 ]]; then - kill $WATCHDOG_PID - rm $DEBUG_FILES_OUTPUT_DIR/watchdog - rm -f $DEBUG_FILES_OUTPUT_DIR/jps-traces.* -fi - -# properly forward exit code -exit $TEST_EXIT_CODE diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh deleted file mode 100755 index 0cb5b8e5d..000000000 --- a/tools/ci/compile.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# -# This file contains tooling for compiling Flink -# - -HERE="`dirname \"$0\"`" # relative -HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized -if [ -z "$HERE" ] ; then - exit 1 # fail -fi -CI_DIR="$HERE/../ci" -MVN_CLEAN_COMPILE_OUT="/tmp/clean_compile.out" - -# source required ci scripts -source "${CI_DIR}/stage.sh" -source "${CI_DIR}/maven-utils.sh" - -echo "Maven version:" -run_mvn -version - -echo "==============================================================================" -echo "Compiling Flink CDC" -echo "==============================================================================" - -EXIT_CODE=0 - -run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests | tee $MVN_CLEAN_COMPILE_OUT - -EXIT_CODE=${PIPESTATUS[0]} - -if [ $EXIT_CODE != 0 ]; then - echo "==============================================================================" - echo "Compiling Flink CDC failed." - echo "==============================================================================" - - grep "0 Unknown Licenses" target/rat.txt > /dev/null - - if [ $? != 0 ]; then - echo "License header check failure detected. Printing first 50 lines for convenience:" - head -n 50 target/rat.txt - fi - - exit $EXIT_CODE -fi - -exit $EXIT_CODE - diff --git a/tools/ci/controller_utils.sh b/tools/ci/controller_utils.sh deleted file mode 100644 index 696f325b2..000000000 --- a/tools/ci/controller_utils.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -print_system_info() { - echo "CPU information" - lscpu - - echo "Memory information" - cat /proc/meminfo - - echo "Disk information" - df -hH - - echo "Running build as" - whoami -} - -print_stacktraces () { - echo "==============================================================================" - echo "The following Java processes are running (JPS)" - echo "==============================================================================" - - JAVA_PROCESSES=`jps` - echo "$JAVA_PROCESSES" - - local pids=( $(echo "$JAVA_PROCESSES" | awk '{print $1}') ) - - for pid in "${pids[@]}"; do - echo "==============================================================================" - echo "Printing stack trace of Java process ${pid}" - echo "==============================================================================" - - jstack $pid - done -} - diff --git a/tools/ci/google-mirror-settings.xml b/tools/ci/google-mirror-settings.xml deleted file mode 100644 index 49e7ccdde..000000000 --- a/tools/ci/google-mirror-settings.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - google-maven-central - GCS Maven Central mirror - https://maven-central-eu.storage-download.googleapis.com/maven2/ - central - - - diff --git a/tools/ci/log4j.properties b/tools/ci/log4j.properties deleted file mode 100644 index 15d17b888..000000000 --- a/tools/ci/log4j.properties +++ /dev/null @@ -1,32 +0,0 @@ -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -rootLogger.level=INFO -rootLogger.appenderRef.out.ref=FileAppender -# ----------------------------------------------------------------------------- -# Console (use 'console') -# ----------------------------------------------------------------------------- -appender.console.name=ConsoleAppender -appender.console.type=CONSOLE -appender.console.layout.type=PatternLayout -appender.console.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n -# ----------------------------------------------------------------------------- -# File (use 'file') -# ----------------------------------------------------------------------------- -appender.file.name=FileAppender -appender.file.type=FILE -appender.file.fileName=${sys:log.dir}/mvn-${sys:mvn.forkNumber}.log -appender.file.layout.type=PatternLayout -appender.file.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n \ No newline at end of file diff --git a/tools/ci/maven-utils.sh b/tools/ci/maven-utils.sh deleted file mode 100755 index 646837ca1..000000000 --- a/tools/ci/maven-utils.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Utility for invoking Maven in CI -function run_mvn { - MVN_CMD="mvn" - if [[ "$M2_HOME" != "" ]]; then - MVN_CMD="${M2_HOME}/bin/mvn" - fi - - ARGS=$@ - INVOCATION="$MVN_CMD $MVN_GLOBAL_OPTIONS $ARGS" - if [[ "$MVN_RUN_VERBOSE" != "false" ]]; then - echo "Invoking mvn with '$INVOCATION'" - fi - eval $INVOCATION -} -export -f run_mvn - -function setup_maven { - set -e # fail if there was an error setting up maven - if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then - wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip - unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip" - rm "apache-maven-${MAVEN_VERSION}-bin.zip" - fi - - export M2_HOME="${MAVEN_VERSIONED_DIR}" - echo "##vso[task.setvariable variable=M2_HOME]$M2_HOME" - - # just in case: clean up the .m2 home and remove invalid jar files - if [ -d "${HOME}/.m2/repository/" ]; then - find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \; - fi - - echo "Installed Maven ${MAVEN_VERSION} to ${M2_HOME}" - set +e -} - -function set_mirror_config { - if [[ "$MAVEN_MIRROR_CONFIG_FILE" != "" ]]; then - echo "[WARN] Maven mirror already configured to $MAVEN_MIRROR_CONFIG_FILE" - exit 0; - fi - - echo "Using Google mirror" - MAVEN_MIRROR_CONFIG_FILE="$CI_DIR/google-mirror-settings.xml" -} - -function collect_coredumps { - local SEARCHDIR=$1 - local TARGET_DIR=$2 - echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'" - for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do - echo "Moving '$file' to target directory ('$TARGET_DIR')" - mv $file $TARGET_DIR/$(echo $file | tr "/" "-") - done -} - -function collect_dmesg { - local TARGET_DIR=$1 - dmesg > $TARGET_DIR/dmesg.out -} - -CI_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -MAVEN_VERSION="3.2.5" -MAVEN_CACHE_DIR=${HOME}/maven_cache -MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION} - - -MAVEN_MIRROR_CONFIG_FILE="" -set_mirror_config - -export MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR="" -# see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html -MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dmaven.wagon.http.pool=false " -# logging -MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn " -# suppress snapshot updates -MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="--no-snapshot-updates " -# enable non-interactive batch mode -MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-B " -# globally control the build profile details -MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="$PROFILE " - -export MVN_GLOBAL_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} " -# use google mirror everywhere -MVN_GLOBAL_OPTIONS+="--settings $MAVEN_MIRROR_CONFIG_FILE " diff --git a/tools/ci/stage.sh b/tools/ci/stage.sh deleted file mode 100755 index 869c4d530..000000000 --- a/tools/ci/stage.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -STAGE_CORE="core" -STAGE_PIPELINE_CONNECTORS="pipeline_connectors" -STAGE_MYSQL="mysql" -STAGE_POSTGRES="postgres" -STAGE_ORACLE="oracle" -STAGE_MONGODB="mongodb" -STAGE_SQLSERVER="sqlserver" -STAGE_TIDB="tidb" -STAGE_OCEANBASE="oceanbase" -STAGE_DB2="db2" -STAGE_VITESS="vitess" -STAGE_E2E="e2e" -STAGE_MISC="misc" - -MODULES_CORE="\ -flink-cdc-cli,\ -flink-cdc-common,\ -flink-cdc-composer,\ -flink-cdc-runtime" - -MODULES_PIPELINE_CONNECTORS="\ -flink-cdc-connect/flink-cdc-pipeline-connectors" - -MODULES_MYSQL="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc" - -MODULES_POSTGRES="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc" - -MODULES_ORACLE="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc" - -MODULES_MONGODB="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc" - -MODULES_SQLSERVER="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc" - -MODULES_TIDB="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc" - -MODULES_OCEANBASE="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc" - -MODULES_DB2="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc" - -MODULES_VITESS="\ -flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\ -flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc" - -MODULES_E2E="\ -flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests,\ -flink-cdc-e2e-tests/flink-cdc-source-e2e-tests" - -function get_compile_modules_for_stage() { - local stage=$1 - - case ${stage} in - (${STAGE_CORE}) - echo "-pl $MODULES_CORE -am" - ;; - (${STAGE_PIPELINE_CONNECTORS}) - echo "-pl $MODULES_PIPELINE_CONNECTORS -am" - ;; - (${STAGE_MYSQL}) - echo "-pl $MODULES_MYSQL -am" - ;; - (${STAGE_POSTGRES}) - echo "-pl $MODULES_POSTGRES -am" - ;; - (${STAGE_ORACLE}) - echo "-pl $MODULES_ORACLE -am" - ;; - (${STAGE_MONGODB}) - echo "-pl $MODULES_MONGODB -am" - ;; - (${STAGE_SQLSERVER}) - echo "-pl $MODULES_SQLSERVER -am" - ;; - (${STAGE_TIDB}) - echo "-pl $MODULES_TIDB -am" - ;; - (${STAGE_OCEANBASE}) - echo "-pl $MODULES_OCEANBASE -am" - ;; - (${STAGE_DB2}) - echo "-pl $MODULES_DB2 -am" - ;; - (${STAGE_VITESS}) - echo "-pl $MODULES_VITESS -am" - ;; - (${STAGE_E2E}) - # compile everything; using the -am switch does not work with negated module lists! - # the negation takes precedence, thus not all required modules would be built - echo "" - ;; - (${STAGE_MISC}) - # compile everything; using the -am switch does not work with negated module lists! - # the negation takes precedence, thus not all required modules would be built - echo "" - ;; - esac -} - -function get_test_modules_for_stage() { - local stage=$1 - - local modules_core=$MODULES_CORE - local modules_pipeline_connectors=$MODULES_PIPELINE_CONNECTORS - local modules_mysql=$MODULES_MYSQL - local modules_postgres=$MODULES_POSTGRES - local modules_oracle=$MODULES_ORACLE - local modules_mongodb=$MODULES_MONGODB - local modules_sqlserver=$MODULES_SQLSERVER - local modules_tidb=$MODULES_TIDB - local modules_oceanbase=$MODULES_OCEANBASE - local modules_db2=$MODULES_DB2 - local modules_vitess=$MODULES_VITESS - local modules_e2e=$MODULES_E2E - local negated_mysql=\!${MODULES_MYSQL//,/,\!} - local negated_postgres=\!${MODULES_POSTGRES//,/,\!} - local negated_oracle=\!${MODULES_ORACLE//,/,\!} - local negated_mongodb=\!${MODULES_MONGODB//,/,\!} - local negated_sqlserver=\!${MODULES_SQLSERVER//,/,\!} - local negated_tidb=\!${MODULES_TIDB//,/,\!} - local negated_oceanbase=\!${MODULES_OCEANBASE//,/,\!} - local negated_db2=\!${MODULES_DB2//,/,\!} - local negated_vitess=\!${MODULES_vitess//,/,\!} - local negated_e2e=\!${MODULES_E2E//,/,\!} - local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb,$negated_sqlserver,$negated_tidb,$negated_oceanbase,$negated_db2,$negated_vitess,$negated_e2e" - - case ${stage} in - (${STAGE_CORE}) - echo "-pl $modules_core" - ;; - (${STAGE_PIPELINE_CONNECTORS}) - echo "-pl $modules_pipeline_connectors" - ;; - (${STAGE_MYSQL}) - echo "-pl $modules_mysql" - ;; - (${STAGE_POSTGRES}) - echo "-pl $modules_postgres" - ;; - (${STAGE_ORACLE}) - echo "-pl $modules_oracle" - ;; - (${STAGE_MONGODB}) - echo "-pl $modules_mongodb" - ;; - (${STAGE_SQLSERVER}) - echo "-pl $modules_sqlserver" - ;; - (${STAGE_TIDB}) - echo "-pl $modules_tidb" - ;; - (${STAGE_OCEANBASE}) - echo "-pl $modules_oceanbase" - ;; - (${STAGE_DB2}) - echo "-pl $modules_db2" - ;; - (${STAGE_VITESS}) - echo "-pl $modules_vitess" - ;; - (${STAGE_E2E}) - echo "-pl $modules_e2e" - ;; - (${STAGE_MISC}) - echo "-pl $modules_misc" - ;; - esac -} diff --git a/tools/ci/test_controller.sh b/tools/ci/test_controller.sh deleted file mode 100755 index 66de5a186..000000000 --- a/tools/ci/test_controller.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# -# This file contains generic control over the test execution. -# - -HERE="`dirname \"$0\"`" # relative -HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized -if [ -z "$HERE" ] ; then - exit 1 -fi - -source "${HERE}/stage.sh" -source "${HERE}/maven-utils.sh" -source "${HERE}/controller_utils.sh" - -STAGE=$1 - -# ============================================================================= -# Step 0: Check & print environment information & configure env -# ============================================================================= - -# check preconditions -if [ -z "$DEBUG_FILES_OUTPUT_DIR" ] ; then - echo "ERROR: Environment variable 'DEBUG_FILES_OUTPUT_DIR' is not set but expected by test_controller.sh. Tests may use this location to store debugging files." - exit 1 -fi - -if [ ! -d "$DEBUG_FILES_OUTPUT_DIR" ] ; then - echo "ERROR: Environment variable DEBUG_FILES_OUTPUT_DIR=$DEBUG_FILES_OUTPUT_DIR points to a directory that does not exist" - exit 1 -fi - -if [ -z "$STAGE" ] ; then - echo "ERROR: Environment variable 'STAGE' is not set but expected by test_controller.sh. The variable refers to the stage being executed." - exit 1 -fi - -echo "Printing environment information" - -echo "PATH=$PATH" -run_mvn -version -echo "Commit: $(git rev-parse HEAD)" -print_system_info - -# enable coredumps for this process -ulimit -c unlimited - -# configure JVMs to produce heap dumps -export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" - -# some tests provide additional logs if they find this variable -export IS_CI=true - -export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log" - -source "${HERE}/watchdog.sh" - -# ============================================================================= -# Step 1: Rebuild jars and install Flink to local maven repository -# ============================================================================= - -LOG4J_PROPERTIES=${HERE}/log4j.properties -MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES" - -MVN_COMMON_OPTIONS="-Dfast $MVN_LOGGING_OPTIONS" -MVN_COMPILE_OPTIONS="-DskipTests" -MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE}) - -CALLBACK_ON_TIMEOUT="print_stacktraces | tee ${DEBUG_FILES_OUTPUT_DIR}/jps-traces.out" -run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install" $CALLBACK_ON_TIMEOUT -EXIT_CODE=$? - -if [ $EXIT_CODE != 0 ]; then - echo "==============================================================================" - echo "Compilation failure detected, skipping test execution." - echo "==============================================================================" - exit $EXIT_CODE -fi - - -# ============================================================================= -# Step 2: Run tests -# ============================================================================= - -MVN_TEST_MODULES=$(get_test_modules_for_stage ${STAGE}) - -run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $PROFILE $MVN_TEST_MODULES verify" $CALLBACK_ON_TIMEOUT -EXIT_CODE=$? - -# ============================================================================= -# Step 3: Put extra logs into $DEBUG_FILES_OUTPUT_DIR -# ============================================================================= - -collect_coredumps $(pwd) $DEBUG_FILES_OUTPUT_DIR - -# Exit code for CI build success/failure -exit $EXIT_CODE diff --git a/tools/ci/watchdog.sh b/tools/ci/watchdog.sh deleted file mode 100755 index 5876db190..000000000 --- a/tools/ci/watchdog.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# -# This file contains a watchdog tool to monitor a task and potentially kill it after -# not producing any output for $MAX_NO_OUTPUT seconds. -# - -# Number of seconds w/o output before printing a stack trace and killing the watched process -MAX_NO_OUTPUT=${MAX_NO_OUTPUT:-900} - -# Number of seconds to sleep before checking the output again -SLEEP_TIME=${SLEEP_TIME:-20} - -# Internal fields -CMD_OUT="/tmp/watchdog.out" -CMD_PID="/tmp/watchdog.pid" -CMD_EXIT="/tmp/watchdog.exit" - - -# ============================================= -# Utility functions -# ============================================= - -max_of() { - local max number - - max="$1" - - for number in "${@:2}"; do - if ((number > max)); then - max="$number" - fi - done - - printf '%d\n' "$max" -} - -# Returns the highest modification time out of $CMD_OUT (which is the command output file) -# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests) -mod_time () { - CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT` - ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null` - echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES` -} - -the_time() { - echo `date +%s` -} - -# watchdog process - -watchdog () { - touch $CMD_OUT - - while true; do - sleep $SLEEP_TIME - - time_diff=$((`the_time` - `mod_time`)) - - if [ $time_diff -ge $MAX_NO_OUTPUT ]; then - echo "==============================================================================" - echo "Process produced no output for ${MAX_NO_OUTPUT} seconds." - echo "==============================================================================" - - # run timeout callback - $CALLBACK_ON_TIMEOUT - - echo "Killing process with pid=$(<$CMD_PID) and all descendants" - pkill -P $(<$CMD_PID) # kill descendants - kill $(<$CMD_PID) # kill process itself - - exit 1 - fi - done -} - - -# ============================================= -# main function -# ============================================= - -# entrypoint -function run_with_watchdog() { - local cmd="$1" - local CALLBACK_ON_TIMEOUT="$2" - - watchdog & - WD_PID=$! - echo "STARTED watchdog (${WD_PID})." - - echo "RUNNING '${cmd}'." - - # Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to - # allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains - # the exit code. This is important for CI build life-cycle (success/failure). - ( $cmd & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT - - EXIT_CODE=$(<$CMD_EXIT) - - echo "Process exited with EXIT CODE: ${EXIT_CODE}." - - # Make sure to kill the watchdog in any case after $CMD has completed - echo "Trying to KILL watchdog (${WD_PID})." - ( kill $WD_PID 2>&1 ) > /dev/null - - rm $CMD_PID - rm $CMD_EXIT - - return $EXIT_CODE -} - -