diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 10cd5ea49..8a6573828 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,19 +21,41 @@ trigger: branches: include: - - '*' # must quote since "*" is a YAML reserved character; we want a string + - '*' # must quote since "*" is a YAML reserved character; we want a string +# Define variables: +# - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching +# - See https://stackoverflow.com/questions/60742105/how-can-i-access-a-secret-value-from-an-azure-pipelines-expression +# to understand why the secrets are handled like this +variables: + MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository + E2E_CACHE_FOLDER: $(Pipeline.Workspace)/e2e_cache + E2E_TARBALL_CACHE: $(Pipeline.Workspace)/e2e_artifact_cache + MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' + CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/** + CACHE_FALLBACK_KEY: maven | $(Agent.OS) + FLINK_ARTIFACT_DIR: $(Pipeline.Workspace)/flink_artifact -pool: - vmImage: 'ubuntu-18.04' +stages: + # CI / PR triggered stage: + - stage: ci + displayName: "CI build (custom builders)" + jobs: + - template: tools/azure-pipelines/jobs-template.yml + parameters: # see template file for a definition of the parameters. + stage_name: ci_build + test_pool_definition: + vmImage: 'ubuntu-20.04' + run_end_to_end: false + jdk: 8 -steps: -- task: Maven@3 - inputs: - mavenPomFile: 'pom.xml' - mavenOptions: '-Xmx3072m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120' - javaHomeOption: 'JDKVersion' - jdkVersionOption: '1.8' - jdkArchitectureOption: 'x64' - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - goals: 'clean verify' +#steps: +# - task: Maven@3 +# inputs: +# mavenPomFile: 'pom.xml' +# mavenOptions: '-Xmx3072m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120' +# javaHomeOption: 'JDKVersion' +# jdkVersionOption: '1.8' +# jdkArchitectureOption: 'x64' +# publishJUnitResults: true +# testResultsFiles: '**/surefire-reports/TEST-*.xml' +# goals: 'clean verify' diff --git a/pom.xml b/pom.xml index 4162fb667..80668f2d7 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,18 @@ under the License. + + + org.apache.maven.plugins + maven-site-plugin + + + attach-descriptor + none + + + + org.apache.rat apache-rat-plugin diff --git a/tools/azure-pipelines/create_build_artifact.sh b/tools/azure-pipelines/create_build_artifact.sh new file mode 100755 index 000000000..b0b15d839 --- /dev/null +++ b/tools/azure-pipelines/create_build_artifact.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +echo "Creating build artifact dir $FLINK_ARTIFACT_DIR" + +cp -r . "$FLINK_ARTIFACT_DIR" + +echo "Minimizing artifact files" + +# reduces the size of the artifact directory to speed up +# the packing&upload / download&unpacking process +# by removing files not required for subsequent stages + +# jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) +find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf + +# .git directory +# not deleting this can cause build stability issues +# merging the cached version sometimes fails +rm -rf "$FLINK_ARTIFACT_DIR/.git" + diff --git a/tools/azure-pipelines/debug_files_utils.sh b/tools/azure-pipelines/debug_files_utils.sh new file mode 100755 index 000000000..50c1b4c5a --- /dev/null +++ b/tools/azure-pipelines/debug_files_utils.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +function prepare_debug_files { + MODULE=$@ + export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files" + export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r' '_')-$(date +%s)" + echo "##vso[task.setvariable variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR" + echo "##vso[task.setvariable variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME" + mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; } +} diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml new file mode 100644 index 000000000..4f6b62440 --- /dev/null +++ b/tools/azure-pipelines/jobs-template.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +parameters: + test_pool_definition: # defines the hardware pool for compilation and unit test execution. + stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage) + run_end_to_end: # if set to 'true', the end to end tests will be executed + jdk: # the jdk version to use + +jobs: + - job: compile_${{parameters.stage_name}} + # succeeded() is needed to allow job cancellation + condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) + pool: ${{parameters.test_pool_definition}} + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all # this cleans the entire workspace directory before running a new job + # It is necessary because the custom build machines are reused for tests. + # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace + + steps: + # The cache task is persisting the .m2 directory between builds, so that + # we do not have to re-download all dependencies from maven central for + # each build. The hope is that downloading the cache is faster than + # all dependencies individually. + # In this configuration, we use a hash over all committed (not generated) .pom files + # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash + # (usually because a pom file has changed), we'll fall back to a key without + # the pom files (CACHE_FALLBACK_KEY). + # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops + - task: Cache@2 + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true # continue the build even if the cache fails. + displayName: Cache Maven local repo + - script: | + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" + echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" + displayName: "Set JDK" + # Compile + - script: | + ./tools/ci/compile.sh || exit $? + ./tools/azure-pipelines/create_build_artifact.sh + displayName: Compile + + # upload artifacts for next stage + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(FLINK_ARTIFACT_DIR) + artifact: FlinkCompileArtifact-${{parameters.stage_name}} + + - job: test_${{parameters.stage_name}} + dependsOn: compile_${{parameters.stage_name}} + condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) + pool: ${{parameters.test_pool_definition}} + timeoutInMinutes: 240 + cancelTimeoutInMinutes: 1 + workspace: + clean: all + strategy: + matrix: + mysql: + module: mysql + postgres: + module: postgres + oracle: + module: oracle + mongodb: + module: mongodb + misc: + module: misc + steps: + # download artifact from compile stage + - task: DownloadPipelineArtifact@2 + inputs: + path: $(FLINK_ARTIFACT_DIR) + artifact: FlinkCompileArtifact-${{parameters.stage_name}} + + - script: ./tools/azure-pipelines/unpack_build_artifact.sh + displayName: "Unpack Build artifact" + + - task: Cache@2 + inputs: + key: $(CACHE_KEY) + restoreKeys: $(CACHE_FALLBACK_KEY) + path: $(MAVEN_CACHE_FOLDER) + continueOnError: true # continue the build even if the cache fails. + condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) + displayName: Cache Maven local repo + + - script: | + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" + echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" + displayName: "Set JDK" + + - script: sudo sysctl -w kernel.core_pattern=core.%p + displayName: Set coredump pattern + + # Test + - script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) + displayName: Test - $(module) + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFormat: 'JUnit' + + # upload debug artifacts + - task: PublishPipelineArtifact@1 + condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', '')) + displayName: Upload Logs + inputs: + targetPath: $(DEBUG_FILES_OUTPUT_DIR) + artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) \ No newline at end of file diff --git a/tools/azure-pipelines/unpack_build_artifact.sh b/tools/azure-pipelines/unpack_build_artifact.sh new file mode 100755 index 000000000..a0a693a90 --- /dev/null +++ b/tools/azure-pipelines/unpack_build_artifact.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +if ! [ -e $FLINK_ARTIFACT_DIR ]; then + echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build." + exit 1 +fi + +echo "Merging cache" +cp -RT "$FLINK_ARTIFACT_DIR" "." + +echo "Adjusting timestamps" + +# adjust timestamps to prevent recompilation +find . -type f -name '*.java' | xargs touch +find . -type f -name '*.class' | xargs touch diff --git a/tools/azure-pipelines/uploading_watchdog.sh b/tools/azure-pipelines/uploading_watchdog.sh new file mode 100755 index 000000000..c49f09320 --- /dev/null +++ b/tools/azure-pipelines/uploading_watchdog.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file has the following tasks +# a) It reads the e2e timeout from the configuration file +# b) It prints a warning if the test has reached 80% of it's execution time +# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts + +COMMAND=$@ + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized +if [ -z "$HERE" ] ; then + exit 1 +fi + +source "${HERE}/../ci/controller_utils.sh" + +source ./tools/azure-pipelines/debug_files_utils.sh +prepare_debug_files "$AGENT_JOBNAME" +export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs" +mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; } +sudo apt-get install -y moreutils + +REAL_START_SECONDS=$(date +"%s") +REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT minutes" +"%s") +REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS)) +KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60)) + +echo "Running command '$COMMAND' with a timeout of $(($REAL_TIMEOUT_SECONDS / 60)) minutes." + +MAIN_PID_FILE="/tmp/uploading_watchdog_main.pid" + +function timeout_watchdog() { + # 95% + sleep $(($REAL_TIMEOUT_SECONDS * 95 / 100)) + echo "==========================================================================================" + echo "=== WARNING: This task took already 95% of the available time budget of $((REAL_TIMEOUT_SECONDS / 60)) minutes ===" + echo "==========================================================================================" + print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.0" + + # final stack trace and kill processes 1 min before timeout + local secondsToKill=$(($REAL_END_SECONDS - $(date +"%s") - $KILL_SECONDS_BEFORE_TIMEOUT)) + if [[ $secondsToKill -lt 0 ]]; then + secondsToKill=0 + fi + sleep $(secondsToKill) + print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.1" + + echo "=============================" + echo "=== WARNING: Killing task ===" + echo "=============================" + pkill -P $(<$MAIN_PID_FILE) # kill descendants + kill $(<$MAIN_PID_FILE) # kill process itself + + exit 42 +} + +timeout_watchdog & +WATCHDOG_PID=$! + +# ts from moreutils prepends the time to each line +( $COMMAND & PID=$! ; echo $PID >$MAIN_PID_FILE ; wait $PID ) | ts | tee $DEBUG_FILES_OUTPUT_DIR/watchdog +TEST_EXIT_CODE=${PIPESTATUS[0]} + +# successful execution, cleanup watchdog related things +if [[ "$TEST_EXIT_CODE" == 0 ]]; then + kill $WATCHDOG_PID + rm $DEBUG_FILES_OUTPUT_DIR/watchdog + rm -f $DEBUG_FILES_OUTPUT_DIR/jps-traces.* +fi + +# properly forward exit code +exit $TEST_EXIT_CODE diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh new file mode 100755 index 000000000..9793db5dd --- /dev/null +++ b/tools/ci/compile.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# +# This file contains tooling for compiling Flink +# + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized +if [ -z "$HERE" ] ; then + exit 1 # fail +fi +CI_DIR="$HERE/../ci" +MVN_CLEAN_COMPILE_OUT="/tmp/clean_compile.out" + +# source required ci scripts +source "${CI_DIR}/stage.sh" +source "${CI_DIR}/maven-utils.sh" + +echo "Maven version:" +run_mvn -version + +echo "==============================================================================" +echo "Compiling Flink CDC" +echo "==============================================================================" + +EXIT_CODE=0 + +run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests | tee $MVN_CLEAN_COMPILE_OUT + +EXIT_CODE=${PIPESTATUS[0]} + +if [ $EXIT_CODE != 0 ]; then + echo "==============================================================================" + echo "Compiling Flink CDC failed." + echo "==============================================================================" + + grep "0 Unknown Licenses" target/rat.txt > /dev/null + + if [ $? != 0 ]; then + echo "License header check failure detected. Printing first 50 lines for convenience:" + head -n 50 target/rat.txt + fi + + exit $EXIT_CODE +fi + +exit $EXIT_CODE + diff --git a/tools/ci/controller_utils.sh b/tools/ci/controller_utils.sh new file mode 100644 index 000000000..3614075da --- /dev/null +++ b/tools/ci/controller_utils.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +print_system_info() { + echo "CPU information" + lscpu + + echo "Memory information" + cat /proc/meminfo + + echo "Disk information" + df -hH + + echo "Running build as" + whoami +} + +print_stacktraces () { + echo "==============================================================================" + echo "The following Java processes are running (JPS)" + echo "==============================================================================" + + JAVA_PROCESSES=`jps` + echo "$JAVA_PROCESSES" + + local pids=( $(echo "$JAVA_PROCESSES" | awk '{print $1}') ) + + for pid in "${pids[@]}"; do + echo "==============================================================================" + echo "Printing stack trace of Java process ${pid}" + echo "==============================================================================" + + jstack $pid + done +} + diff --git a/tools/ci/google-mirror-settings.xml b/tools/ci/google-mirror-settings.xml new file mode 100644 index 000000000..49a3b7133 --- /dev/null +++ b/tools/ci/google-mirror-settings.xml @@ -0,0 +1,28 @@ + + + + + google-maven-central + GCS Maven Central mirror + https://maven-central-eu.storage-download.googleapis.com/maven2/ + central + + + diff --git a/tools/ci/log4j.properties b/tools/ci/log4j.properties new file mode 100644 index 000000000..4f1fe4085 --- /dev/null +++ b/tools/ci/log4j.properties @@ -0,0 +1,34 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +rootLogger.level=INFO +rootLogger.appenderRef.out.ref=FileAppender +# ----------------------------------------------------------------------------- +# Console (use 'console') +# ----------------------------------------------------------------------------- +appender.console.name=ConsoleAppender +appender.console.type=CONSOLE +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n +# ----------------------------------------------------------------------------- +# File (use 'file') +# ----------------------------------------------------------------------------- +appender.file.name=FileAppender +appender.file.type=FILE +appender.file.fileName=${sys:log.dir}/mvn-${sys:mvn.forkNumber}.log +appender.file.layout.type=PatternLayout +appender.file.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n \ No newline at end of file diff --git a/tools/ci/maven-utils.sh b/tools/ci/maven-utils.sh new file mode 100755 index 000000000..104401a7a --- /dev/null +++ b/tools/ci/maven-utils.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Utility for invoking Maven in CI +function run_mvn { + MVN_CMD="mvn" + if [[ "$M2_HOME" != "" ]]; then + MVN_CMD="${M2_HOME}/bin/mvn" + fi + + ARGS=$@ + INVOCATION="$MVN_CMD $MVN_GLOBAL_OPTIONS $ARGS" + if [[ "$MVN_RUN_VERBOSE" != "false" ]]; then + echo "Invoking mvn with '$INVOCATION'" + fi + eval $INVOCATION +} +export -f run_mvn + +function setup_maven { + set -e # fail if there was an error setting up maven + if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then + wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip + unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip" + rm "apache-maven-${MAVEN_VERSION}-bin.zip" + fi + + export M2_HOME="${MAVEN_VERSIONED_DIR}" + echo "##vso[task.setvariable variable=M2_HOME]$M2_HOME" + + # just in case: clean up the .m2 home and remove invalid jar files + if [ -d "${HOME}/.m2/repository/" ]; then + find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \; + fi + + echo "Installed Maven ${MAVEN_VERSION} to ${M2_HOME}" + set +e +} + +function set_mirror_config { + if [[ "$MAVEN_MIRROR_CONFIG_FILE" != "" ]]; then + echo "[WARN] Maven mirror already configured to $MAVEN_MIRROR_CONFIG_FILE" + exit 0; + fi + + echo "Using Google mirror" + MAVEN_MIRROR_CONFIG_FILE="$CI_DIR/google-mirror-settings.xml" +} + +function collect_coredumps { + local SEARCHDIR=$1 + local TARGET_DIR=$2 + echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'" + for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do + echo "Moving '$file' to target directory ('$TARGET_DIR')" + mv $file $TARGET_DIR/$(echo $file | tr "/" "-") + done +} + +function collect_dmesg { + local TARGET_DIR=$1 + dmesg > $TARGET_DIR/dmesg.out +} + +CI_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +MAVEN_VERSION="3.2.5" +MAVEN_CACHE_DIR=${HOME}/maven_cache +MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION} + + +MAVEN_MIRROR_CONFIG_FILE="" +set_mirror_config + +export MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR="" +# see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html +MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dmaven.wagon.http.pool=false " +# logging +MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn " +# suppress snapshot updates +MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="--no-snapshot-updates " +# enable non-interactive batch mode +MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-B " +# globally control the build profile details +MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="$PROFILE " + +export MVN_GLOBAL_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} " +# use google mirror everywhere +MVN_GLOBAL_OPTIONS+="--settings $MAVEN_MIRROR_CONFIG_FILE " diff --git a/tools/ci/stage.sh b/tools/ci/stage.sh new file mode 100755 index 000000000..3f6bad91e --- /dev/null +++ b/tools/ci/stage.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +STAGE_MYSQL="mysql" +STAGE_POSTGRES="postgres" +STAGE_ORACLE="oracle" +STAGE_MONGODB="mongodb" +STAGE_MISC="misc" + +MODULES_MYSQL="\ +flink-connector-mysql-cdc,\ +flink-sql-connector-mysql-cdc" + +MODULES_POSTGRES="\ +flink-connector-postgres-cdc,\ +flink-sql-connector-postgres-cdc" + +MODULES_ORACLE="\ +flink-connector-oracle-cdc,\ +flink-sql-connector-oracle-cdc" + +MODULES_MONGODB="\ +flink-connector-mongodb-cdc,\ +flink-sql-connector-mongodb-cdc" + +function get_compile_modules_for_stage() { + local stage=$1 + + case ${stage} in + (${STAGE_MYSQL}) + echo "-pl $MODULES_MYSQL -am" + ;; + (${STAGE_POSTGRES}) + echo "-pl $MODULES_POSTGRES -am" + ;; + (${STAGE_ORACLE}) + echo "-pl $MODULES_ORACLE -am" + ;; + (${STAGE_MONGODB}) + echo "-pl $MODULES_MONGODB -am" + ;; + (${STAGE_MISC}) + # compile everything; using the -am switch does not work with negated module lists! + # the negation takes precedence, thus not all required modules would be built + echo "" + ;; + esac +} + +function get_test_modules_for_stage() { + local stage=$1 + + local modules_mysql=$MODULES_MYSQL + local modules_postgres=$MODULES_POSTGRES + local modules_oracle=$MODULES_ORACLE + local modules_mongodb=$MODULES_MONGODB + local negated_mysql=\!${MODULES_MYSQL//,/,\!} + local negated_postgres=\!${MODULES_POSTGRES//,/,\!} + local negated_oracle=\!${MODULES_ORACLE//,/,\!} + local negated_mongodb=\!${MODULES_MONGODB//,/,\!} + local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb" + + case ${stage} in + (${STAGE_MYSQL}) + echo "-pl $modules_mysql" + ;; + (${STAGE_POSTGRES}) + echo "-pl $modules_postgres" + ;; + (${STAGE_ORACLE}) + echo "-pl $modules_oracle" + ;; + (${STAGE_MONGODB}) + echo "-pl $modules_mongodb" + ;; + (${STAGE_MISC}) + echo "-pl $modules_misc" + ;; + esac +} diff --git a/tools/ci/test_controller.sh b/tools/ci/test_controller.sh new file mode 100755 index 000000000..03dfe6e48 --- /dev/null +++ b/tools/ci/test_controller.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# +# This file contains generic control over the test execution. +# + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized +if [ -z "$HERE" ] ; then + exit 1 +fi + +source "${HERE}/stage.sh" +source "${HERE}/maven-utils.sh" +source "${HERE}/controller_utils.sh" + +STAGE=$1 + +# ============================================================================= +# Step 0: Check & print environment information & configure env +# ============================================================================= + +# check preconditions +if [ -z "$DEBUG_FILES_OUTPUT_DIR" ] ; then + echo "ERROR: Environment variable 'DEBUG_FILES_OUTPUT_DIR' is not set but expected by test_controller.sh. Tests may use this location to store debugging files." + exit 1 +fi + +if [ ! -d "$DEBUG_FILES_OUTPUT_DIR" ] ; then + echo "ERROR: Environment variable DEBUG_FILES_OUTPUT_DIR=$DEBUG_FILES_OUTPUT_DIR points to a directory that does not exist" + exit 1 +fi + +if [ -z "$STAGE" ] ; then + echo "ERROR: Environment variable 'STAGE' is not set but expected by test_controller.sh. THe variable refers to the stage being executed." + exit 1 +fi + +echo "Printing environment information" + +echo "PATH=$PATH" +run_mvn -version +echo "Commit: $(git rev-parse HEAD)" +print_system_info + +# enable coredumps for this process +ulimit -c unlimited + +# configure JVMs to produce heap dumps +export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" + +# some tests provide additional logs if they find this variable +export IS_CI=true + +export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log" + +source "${HERE}/watchdog.sh" + +# ============================================================================= +# Step 1: Rebuild jars and install Flink to local maven repository +# ============================================================================= + +LOG4J_PROPERTIES=${HERE}/log4j.properties +MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES" + +MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build $MVN_LOGGING_OPTIONS" +MVN_COMPILE_OPTIONS="-DskipTests" +MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE}) + +CALLBACK_ON_TIMEOUT="print_stacktraces | tee ${DEBUG_FILES_OUTPUT_DIR}/jps-traces.out" +run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install" $CALLBACK_ON_TIMEOUT +EXIT_CODE=$? + +if [ $EXIT_CODE != 0 ]; then + echo "==============================================================================" + echo "Compilation failure detected, skipping test execution." + echo "==============================================================================" + exit $EXIT_CODE +fi + + +# ============================================================================= +# Step 2: Run tests +# ============================================================================= + +MVN_TEST_OPTIONS="-Dflink.tests.with-openssl" +MVN_TEST_MODULES=$(get_test_modules_for_stage ${STAGE}) + +run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify" $CALLBACK_ON_TIMEOUT +EXIT_CODE=$? + +# ============================================================================= +# Step 3: Put extra logs into $DEBUG_FILES_OUTPUT_DIR +# ============================================================================= + +collect_coredumps $(pwd) $DEBUG_FILES_OUTPUT_DIR + +# Exit code for CI build success/failure +exit $EXIT_CODE diff --git a/tools/ci/watchdog.sh b/tools/ci/watchdog.sh new file mode 100755 index 000000000..1c069e388 --- /dev/null +++ b/tools/ci/watchdog.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# +# This file contains a watchdog tool to monitor a task and potentially kill it after +# not producing any output for $MAX_NO_OUTPUT seconds. +# + +# Number of seconds w/o output before printing a stack trace and killing the watched process +MAX_NO_OUTPUT=${MAX_NO_OUTPUT:-900} + +# Number of seconds to sleep before checking the output again +SLEEP_TIME=${SLEEP_TIME:-20} + +# Internal fields +CMD_OUT="/tmp/watchdog.out" +CMD_PID="/tmp/watchdog.pid" +CMD_EXIT="/tmp/watchdog.exit" + + +# ============================================= +# Utility functions +# ============================================= + +max_of() { + local max number + + max="$1" + + for number in "${@:2}"; do + if ((number > max)); then + max="$number" + fi + done + + printf '%d\n' "$max" +} + +# Returns the highest modification time out of $CMD_OUT (which is the command output file) +# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests) +mod_time () { + CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT` + ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null` + echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES` +} + +the_time() { + echo `date +%s` +} + +# watchdog process + +watchdog () { + touch $CMD_OUT + + while true; do + sleep $SLEEP_TIME + + time_diff=$((`the_time` - `mod_time`)) + + if [ $time_diff -ge $MAX_NO_OUTPUT ]; then + echo "==============================================================================" + echo "Process produced no output for ${MAX_NO_OUTPUT} seconds." + echo "==============================================================================" + + # run timeout callback + $CALLBACK_ON_TIMEOUT + + echo "Killing process with pid=$(<$CMD_PID) and all descendants" + pkill -P $(<$CMD_PID) # kill descendants + kill $(<$CMD_PID) # kill process itself + + exit 1 + fi + done +} + + +# ============================================= +# main function +# ============================================= + +# entrypoint +function run_with_watchdog() { + local cmd="$1" + local CALLBACK_ON_TIMEOUT="$2" + + watchdog & + WD_PID=$! + echo "STARTED watchdog (${WD_PID})." + + echo "RUNNING '${cmd}'." + + # Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to + # allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains + # the exit code. This is important for CI build life-cycle (success/failure). + ( $cmd & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT + + EXIT_CODE=$(<$CMD_EXIT) + + echo "Process exited with EXIT CODE: ${EXIT_CODE}." + + # Make sure to kill the watchdog in any case after $CMD has completed + echo "Trying to KILL watchdog (${WD_PID})." + ( kill $WD_PID 2>&1 ) > /dev/null + + rm $CMD_PID + rm $CMD_EXIT + + return $EXIT_CODE +} + +