[tests] Split the CI pipeline into multiple stages for better performance (#572)
Jark Wu
3 years ago
committed by
GitHub
parent
5f3393c7f6
commit
8ba9b70e88
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
echo "Creating build artifact dir $FLINK_ARTIFACT_DIR"
|
||||
|
||||
cp -r . "$FLINK_ARTIFACT_DIR"
|
||||
|
||||
echo "Minimizing artifact files"
|
||||
|
||||
# reduces the size of the artifact directory to speed up
|
||||
# the packing&upload / download&unpacking process
|
||||
# by removing files not required for subsequent stages
|
||||
|
||||
# jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
|
||||
find "$FLINK_ARTIFACT_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf
|
||||
|
||||
# .git directory
|
||||
# not deleting this can cause build stability issues
|
||||
# merging the cached version sometimes fails
|
||||
rm -rf "$FLINK_ARTIFACT_DIR/.git"
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
function prepare_debug_files {
|
||||
MODULE=$@
|
||||
export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files"
|
||||
export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r' '_')-$(date +%s)"
|
||||
echo "##vso[task.setvariable variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR"
|
||||
echo "##vso[task.setvariable variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME"
|
||||
mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; }
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
parameters:
|
||||
test_pool_definition: # defines the hardware pool for compilation and unit test execution.
|
||||
stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage)
|
||||
run_end_to_end: # if set to 'true', the end to end tests will be executed
|
||||
jdk: # the jdk version to use
|
||||
|
||||
jobs:
|
||||
- job: compile_${{parameters.stage_name}}
|
||||
# succeeded() is needed to allow job cancellation
|
||||
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
|
||||
pool: ${{parameters.test_pool_definition}}
|
||||
timeoutInMinutes: 240
|
||||
cancelTimeoutInMinutes: 1
|
||||
workspace:
|
||||
clean: all # this cleans the entire workspace directory before running a new job
|
||||
# It is necessary because the custom build machines are reused for tests.
|
||||
# See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
|
||||
|
||||
steps:
|
||||
# The cache task is persisting the .m2 directory between builds, so that
|
||||
# we do not have to re-download all dependencies from maven central for
|
||||
# each build. The hope is that downloading the cache is faster than
|
||||
# all dependencies individually.
|
||||
# In this configuration, we use a hash over all committed (not generated) .pom files
|
||||
# as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash
|
||||
# (usually because a pom file has changed), we'll fall back to a key without
|
||||
# the pom files (CACHE_FALLBACK_KEY).
|
||||
# Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: $(CACHE_KEY)
|
||||
restoreKeys: $(CACHE_FALLBACK_KEY)
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
continueOnError: true # continue the build even if the cache fails.
|
||||
displayName: Cache Maven local repo
|
||||
- script: |
|
||||
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
|
||||
echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
|
||||
displayName: "Set JDK"
|
||||
# Compile
|
||||
- script: |
|
||||
./tools/ci/compile.sh || exit $?
|
||||
./tools/azure-pipelines/create_build_artifact.sh
|
||||
displayName: Compile
|
||||
|
||||
# upload artifacts for next stage
|
||||
- task: PublishPipelineArtifact@1
|
||||
inputs:
|
||||
targetPath: $(FLINK_ARTIFACT_DIR)
|
||||
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
|
||||
|
||||
- job: test_${{parameters.stage_name}}
|
||||
dependsOn: compile_${{parameters.stage_name}}
|
||||
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
|
||||
pool: ${{parameters.test_pool_definition}}
|
||||
timeoutInMinutes: 240
|
||||
cancelTimeoutInMinutes: 1
|
||||
workspace:
|
||||
clean: all
|
||||
strategy:
|
||||
matrix:
|
||||
mysql:
|
||||
module: mysql
|
||||
postgres:
|
||||
module: postgres
|
||||
oracle:
|
||||
module: oracle
|
||||
mongodb:
|
||||
module: mongodb
|
||||
misc:
|
||||
module: misc
|
||||
steps:
|
||||
# download artifact from compile stage
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
path: $(FLINK_ARTIFACT_DIR)
|
||||
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
|
||||
|
||||
- script: ./tools/azure-pipelines/unpack_build_artifact.sh
|
||||
displayName: "Unpack Build artifact"
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: $(CACHE_KEY)
|
||||
restoreKeys: $(CACHE_FALLBACK_KEY)
|
||||
path: $(MAVEN_CACHE_FOLDER)
|
||||
continueOnError: true # continue the build even if the cache fails.
|
||||
condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
|
||||
displayName: Cache Maven local repo
|
||||
|
||||
- script: |
|
||||
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
|
||||
echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
|
||||
displayName: "Set JDK"
|
||||
|
||||
- script: sudo sysctl -w kernel.core_pattern=core.%p
|
||||
displayName: Set coredump pattern
|
||||
|
||||
# Test
|
||||
- script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module)
|
||||
displayName: Test - $(module)
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: succeededOrFailed()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit'
|
||||
|
||||
# upload debug artifacts
|
||||
- task: PublishPipelineArtifact@1
|
||||
condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', ''))
|
||||
displayName: Upload Logs
|
||||
inputs:
|
||||
targetPath: $(DEBUG_FILES_OUTPUT_DIR)
|
||||
artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME)
|
||||
@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
|
||||
if ! [ -e $FLINK_ARTIFACT_DIR ]; then
|
||||
echo "Cached flink dir $FLINK_ARTIFACT_DIR does not exist. Exiting build."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Merging cache"
|
||||
cp -RT "$FLINK_ARTIFACT_DIR" "."
|
||||
|
||||
echo "Adjusting timestamps"
|
||||
|
||||
# adjust timestamps to prevent recompilation
|
||||
find . -type f -name '*.java' | xargs touch
|
||||
find . -type f -name '*.class' | xargs touch
|
||||
@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This file has the following tasks
|
||||
# a) It reads the e2e timeout from the configuration file
|
||||
# b) It prints a warning if the test has reached 80% of it's execution time
|
||||
# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts
|
||||
|
||||
COMMAND=$@
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${HERE}/../ci/controller_utils.sh"
|
||||
|
||||
source ./tools/azure-pipelines/debug_files_utils.sh
|
||||
prepare_debug_files "$AGENT_JOBNAME"
|
||||
export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
|
||||
mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; }
|
||||
sudo apt-get install -y moreutils
|
||||
|
||||
REAL_START_SECONDS=$(date +"%s")
|
||||
REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT minutes" +"%s")
|
||||
REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS))
|
||||
KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60))
|
||||
|
||||
echo "Running command '$COMMAND' with a timeout of $(($REAL_TIMEOUT_SECONDS / 60)) minutes."
|
||||
|
||||
MAIN_PID_FILE="/tmp/uploading_watchdog_main.pid"
|
||||
|
||||
function timeout_watchdog() {
|
||||
# 95%
|
||||
sleep $(($REAL_TIMEOUT_SECONDS * 95 / 100))
|
||||
echo "=========================================================================================="
|
||||
echo "=== WARNING: This task took already 95% of the available time budget of $((REAL_TIMEOUT_SECONDS / 60)) minutes ==="
|
||||
echo "=========================================================================================="
|
||||
print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.0"
|
||||
|
||||
# final stack trace and kill processes 1 min before timeout
|
||||
local secondsToKill=$(($REAL_END_SECONDS - $(date +"%s") - $KILL_SECONDS_BEFORE_TIMEOUT))
|
||||
if [[ $secondsToKill -lt 0 ]]; then
|
||||
secondsToKill=0
|
||||
fi
|
||||
sleep $(secondsToKill)
|
||||
print_stacktraces | tee "$DEBUG_FILES_OUTPUT_DIR/jps-traces.1"
|
||||
|
||||
echo "============================="
|
||||
echo "=== WARNING: Killing task ==="
|
||||
echo "============================="
|
||||
pkill -P $(<$MAIN_PID_FILE) # kill descendants
|
||||
kill $(<$MAIN_PID_FILE) # kill process itself
|
||||
|
||||
exit 42
|
||||
}
|
||||
|
||||
timeout_watchdog &
|
||||
WATCHDOG_PID=$!
|
||||
|
||||
# ts from moreutils prepends the time to each line
|
||||
( $COMMAND & PID=$! ; echo $PID >$MAIN_PID_FILE ; wait $PID ) | ts | tee $DEBUG_FILES_OUTPUT_DIR/watchdog
|
||||
TEST_EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
# successful execution, cleanup watchdog related things
|
||||
if [[ "$TEST_EXIT_CODE" == 0 ]]; then
|
||||
kill $WATCHDOG_PID
|
||||
rm $DEBUG_FILES_OUTPUT_DIR/watchdog
|
||||
rm -f $DEBUG_FILES_OUTPUT_DIR/jps-traces.*
|
||||
fi
|
||||
|
||||
# properly forward exit code
|
||||
exit $TEST_EXIT_CODE
|
||||
@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains tooling for compiling Flink
|
||||
#
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1 # fail
|
||||
fi
|
||||
CI_DIR="$HERE/../ci"
|
||||
MVN_CLEAN_COMPILE_OUT="/tmp/clean_compile.out"
|
||||
|
||||
# source required ci scripts
|
||||
source "${CI_DIR}/stage.sh"
|
||||
source "${CI_DIR}/maven-utils.sh"
|
||||
|
||||
echo "Maven version:"
|
||||
run_mvn -version
|
||||
|
||||
echo "=============================================================================="
|
||||
echo "Compiling Flink CDC"
|
||||
echo "=============================================================================="
|
||||
|
||||
EXIT_CODE=0
|
||||
|
||||
run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests | tee $MVN_CLEAN_COMPILE_OUT
|
||||
|
||||
EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
if [ $EXIT_CODE != 0 ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Compiling Flink CDC failed."
|
||||
echo "=============================================================================="
|
||||
|
||||
grep "0 Unknown Licenses" target/rat.txt > /dev/null
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "License header check failure detected. Printing first 50 lines for convenience:"
|
||||
head -n 50 target/rat.txt
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
print_system_info() {
|
||||
echo "CPU information"
|
||||
lscpu
|
||||
|
||||
echo "Memory information"
|
||||
cat /proc/meminfo
|
||||
|
||||
echo "Disk information"
|
||||
df -hH
|
||||
|
||||
echo "Running build as"
|
||||
whoami
|
||||
}
|
||||
|
||||
print_stacktraces () {
|
||||
echo "=============================================================================="
|
||||
echo "The following Java processes are running (JPS)"
|
||||
echo "=============================================================================="
|
||||
|
||||
JAVA_PROCESSES=`jps`
|
||||
echo "$JAVA_PROCESSES"
|
||||
|
||||
local pids=( $(echo "$JAVA_PROCESSES" | awk '{print $1}') )
|
||||
|
||||
for pid in "${pids[@]}"; do
|
||||
echo "=============================================================================="
|
||||
echo "Printing stack trace of Java process ${pid}"
|
||||
echo "=============================================================================="
|
||||
|
||||
jstack $pid
|
||||
done
|
||||
}
|
||||
|
||||
@ -0,0 +1,28 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
<settings>
|
||||
<mirrors>
|
||||
<mirror>
|
||||
<id>google-maven-central</id>
|
||||
<name>GCS Maven Central mirror</name>
|
||||
<url>https://maven-central-eu.storage-download.googleapis.com/maven2/</url>
|
||||
<mirrorOf>central</mirrorOf>
|
||||
</mirror>
|
||||
</mirrors>
|
||||
</settings>
|
||||
@ -0,0 +1,34 @@
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
rootLogger.level=INFO
|
||||
rootLogger.appenderRef.out.ref=FileAppender
|
||||
# -----------------------------------------------------------------------------
|
||||
# Console (use 'console')
|
||||
# -----------------------------------------------------------------------------
|
||||
appender.console.name=ConsoleAppender
|
||||
appender.console.type=CONSOLE
|
||||
appender.console.layout.type=PatternLayout
|
||||
appender.console.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n
|
||||
# -----------------------------------------------------------------------------
|
||||
# File (use 'file')
|
||||
# -----------------------------------------------------------------------------
|
||||
appender.file.name=FileAppender
|
||||
appender.file.type=FILE
|
||||
appender.file.fileName=${sys:log.dir}/mvn-${sys:mvn.forkNumber}.log
|
||||
appender.file.layout.type=PatternLayout
|
||||
appender.file.layout.pattern=%d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n
|
||||
@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Utility for invoking Maven in CI
|
||||
function run_mvn {
|
||||
MVN_CMD="mvn"
|
||||
if [[ "$M2_HOME" != "" ]]; then
|
||||
MVN_CMD="${M2_HOME}/bin/mvn"
|
||||
fi
|
||||
|
||||
ARGS=$@
|
||||
INVOCATION="$MVN_CMD $MVN_GLOBAL_OPTIONS $ARGS"
|
||||
if [[ "$MVN_RUN_VERBOSE" != "false" ]]; then
|
||||
echo "Invoking mvn with '$INVOCATION'"
|
||||
fi
|
||||
eval $INVOCATION
|
||||
}
|
||||
export -f run_mvn
|
||||
|
||||
function setup_maven {
|
||||
set -e # fail if there was an error setting up maven
|
||||
if [ ! -d "${MAVEN_VERSIONED_DIR}" ]; then
|
||||
wget https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip
|
||||
unzip -d "${MAVEN_CACHE_DIR}" -qq "apache-maven-${MAVEN_VERSION}-bin.zip"
|
||||
rm "apache-maven-${MAVEN_VERSION}-bin.zip"
|
||||
fi
|
||||
|
||||
export M2_HOME="${MAVEN_VERSIONED_DIR}"
|
||||
echo "##vso[task.setvariable variable=M2_HOME]$M2_HOME"
|
||||
|
||||
# just in case: clean up the .m2 home and remove invalid jar files
|
||||
if [ -d "${HOME}/.m2/repository/" ]; then
|
||||
find ${HOME}/.m2/repository/ -name "*.jar" -exec sh -c 'if ! zip -T {} >/dev/null ; then echo "deleting invalid file: {}"; rm -f {} ; fi' \;
|
||||
fi
|
||||
|
||||
echo "Installed Maven ${MAVEN_VERSION} to ${M2_HOME}"
|
||||
set +e
|
||||
}
|
||||
|
||||
function set_mirror_config {
|
||||
if [[ "$MAVEN_MIRROR_CONFIG_FILE" != "" ]]; then
|
||||
echo "[WARN] Maven mirror already configured to $MAVEN_MIRROR_CONFIG_FILE"
|
||||
exit 0;
|
||||
fi
|
||||
|
||||
echo "Using Google mirror"
|
||||
MAVEN_MIRROR_CONFIG_FILE="$CI_DIR/google-mirror-settings.xml"
|
||||
}
|
||||
|
||||
function collect_coredumps {
|
||||
local SEARCHDIR=$1
|
||||
local TARGET_DIR=$2
|
||||
echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'"
|
||||
for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do
|
||||
echo "Moving '$file' to target directory ('$TARGET_DIR')"
|
||||
mv $file $TARGET_DIR/$(echo $file | tr "/" "-")
|
||||
done
|
||||
}
|
||||
|
||||
function collect_dmesg {
|
||||
local TARGET_DIR=$1
|
||||
dmesg > $TARGET_DIR/dmesg.out
|
||||
}
|
||||
|
||||
CI_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
MAVEN_VERSION="3.2.5"
|
||||
MAVEN_CACHE_DIR=${HOME}/maven_cache
|
||||
MAVEN_VERSIONED_DIR=${MAVEN_CACHE_DIR}/apache-maven-${MAVEN_VERSION}
|
||||
|
||||
|
||||
MAVEN_MIRROR_CONFIG_FILE=""
|
||||
set_mirror_config
|
||||
|
||||
export MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR=""
|
||||
# see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dmaven.wagon.http.pool=false "
|
||||
# logging
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn "
|
||||
# suppress snapshot updates
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="--no-snapshot-updates "
|
||||
# enable non-interactive batch mode
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="-B "
|
||||
# globally control the build profile details
|
||||
MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR+="$PROFILE "
|
||||
|
||||
export MVN_GLOBAL_OPTIONS="${MVN_GLOBAL_OPTIONS_WITHOUT_MIRROR} "
|
||||
# use google mirror everywhere
|
||||
MVN_GLOBAL_OPTIONS+="--settings $MAVEN_MIRROR_CONFIG_FILE "
|
||||
@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
STAGE_MYSQL="mysql"
|
||||
STAGE_POSTGRES="postgres"
|
||||
STAGE_ORACLE="oracle"
|
||||
STAGE_MONGODB="mongodb"
|
||||
STAGE_MISC="misc"
|
||||
|
||||
MODULES_MYSQL="\
|
||||
flink-connector-mysql-cdc,\
|
||||
flink-sql-connector-mysql-cdc"
|
||||
|
||||
MODULES_POSTGRES="\
|
||||
flink-connector-postgres-cdc,\
|
||||
flink-sql-connector-postgres-cdc"
|
||||
|
||||
MODULES_ORACLE="\
|
||||
flink-connector-oracle-cdc,\
|
||||
flink-sql-connector-oracle-cdc"
|
||||
|
||||
MODULES_MONGODB="\
|
||||
flink-connector-mongodb-cdc,\
|
||||
flink-sql-connector-mongodb-cdc"
|
||||
|
||||
function get_compile_modules_for_stage() {
|
||||
local stage=$1
|
||||
|
||||
case ${stage} in
|
||||
(${STAGE_MYSQL})
|
||||
echo "-pl $MODULES_MYSQL -am"
|
||||
;;
|
||||
(${STAGE_POSTGRES})
|
||||
echo "-pl $MODULES_POSTGRES -am"
|
||||
;;
|
||||
(${STAGE_ORACLE})
|
||||
echo "-pl $MODULES_ORACLE -am"
|
||||
;;
|
||||
(${STAGE_MONGODB})
|
||||
echo "-pl $MODULES_MONGODB -am"
|
||||
;;
|
||||
(${STAGE_MISC})
|
||||
# compile everything; using the -am switch does not work with negated module lists!
|
||||
# the negation takes precedence, thus not all required modules would be built
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function get_test_modules_for_stage() {
|
||||
local stage=$1
|
||||
|
||||
local modules_mysql=$MODULES_MYSQL
|
||||
local modules_postgres=$MODULES_POSTGRES
|
||||
local modules_oracle=$MODULES_ORACLE
|
||||
local modules_mongodb=$MODULES_MONGODB
|
||||
local negated_mysql=\!${MODULES_MYSQL//,/,\!}
|
||||
local negated_postgres=\!${MODULES_POSTGRES//,/,\!}
|
||||
local negated_oracle=\!${MODULES_ORACLE//,/,\!}
|
||||
local negated_mongodb=\!${MODULES_MONGODB//,/,\!}
|
||||
local modules_misc="$negated_mysql,$negated_postgres,$negated_oracle,$negated_mongodb"
|
||||
|
||||
case ${stage} in
|
||||
(${STAGE_MYSQL})
|
||||
echo "-pl $modules_mysql"
|
||||
;;
|
||||
(${STAGE_POSTGRES})
|
||||
echo "-pl $modules_postgres"
|
||||
;;
|
||||
(${STAGE_ORACLE})
|
||||
echo "-pl $modules_oracle"
|
||||
;;
|
||||
(${STAGE_MONGODB})
|
||||
echo "-pl $modules_mongodb"
|
||||
;;
|
||||
(${STAGE_MISC})
|
||||
echo "-pl $modules_misc"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains generic control over the test execution.
|
||||
#
|
||||
|
||||
HERE="`dirname \"$0\"`" # relative
|
||||
HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized
|
||||
if [ -z "$HERE" ] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${HERE}/stage.sh"
|
||||
source "${HERE}/maven-utils.sh"
|
||||
source "${HERE}/controller_utils.sh"
|
||||
|
||||
STAGE=$1
|
||||
|
||||
# =============================================================================
|
||||
# Step 0: Check & print environment information & configure env
|
||||
# =============================================================================
|
||||
|
||||
# check preconditions
|
||||
if [ -z "$DEBUG_FILES_OUTPUT_DIR" ] ; then
|
||||
echo "ERROR: Environment variable 'DEBUG_FILES_OUTPUT_DIR' is not set but expected by test_controller.sh. Tests may use this location to store debugging files."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$DEBUG_FILES_OUTPUT_DIR" ] ; then
|
||||
echo "ERROR: Environment variable DEBUG_FILES_OUTPUT_DIR=$DEBUG_FILES_OUTPUT_DIR points to a directory that does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$STAGE" ] ; then
|
||||
echo "ERROR: Environment variable 'STAGE' is not set but expected by test_controller.sh. THe variable refers to the stage being executed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Printing environment information"
|
||||
|
||||
echo "PATH=$PATH"
|
||||
run_mvn -version
|
||||
echo "Commit: $(git rev-parse HEAD)"
|
||||
print_system_info
|
||||
|
||||
# enable coredumps for this process
|
||||
ulimit -c unlimited
|
||||
|
||||
# configure JVMs to produce heap dumps
|
||||
export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
|
||||
|
||||
# some tests provide additional logs if they find this variable
|
||||
export IS_CI=true
|
||||
|
||||
export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log"
|
||||
|
||||
source "${HERE}/watchdog.sh"
|
||||
|
||||
# =============================================================================
|
||||
# Step 1: Rebuild jars and install Flink to local maven repository
|
||||
# =============================================================================
|
||||
|
||||
LOG4J_PROPERTIES=${HERE}/log4j.properties
|
||||
MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
|
||||
|
||||
MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast -Pskip-webui-build $MVN_LOGGING_OPTIONS"
|
||||
MVN_COMPILE_OPTIONS="-DskipTests"
|
||||
MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${STAGE})
|
||||
|
||||
CALLBACK_ON_TIMEOUT="print_stacktraces | tee ${DEBUG_FILES_OUTPUT_DIR}/jps-traces.out"
|
||||
run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_COMPILE_OPTIONS $PROFILE $MVN_COMPILE_MODULES install" $CALLBACK_ON_TIMEOUT
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE != 0 ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Compilation failure detected, skipping test execution."
|
||||
echo "=============================================================================="
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Step 2: Run tests
|
||||
# =============================================================================
|
||||
|
||||
MVN_TEST_OPTIONS="-Dflink.tests.with-openssl"
|
||||
MVN_TEST_MODULES=$(get_test_modules_for_stage ${STAGE})
|
||||
|
||||
run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify" $CALLBACK_ON_TIMEOUT
|
||||
EXIT_CODE=$?
|
||||
|
||||
# =============================================================================
|
||||
# Step 3: Put extra logs into $DEBUG_FILES_OUTPUT_DIR
|
||||
# =============================================================================
|
||||
|
||||
collect_coredumps $(pwd) $DEBUG_FILES_OUTPUT_DIR
|
||||
|
||||
# Exit code for CI build success/failure
|
||||
exit $EXIT_CODE
|
||||
@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env bash
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
#
|
||||
# This file contains a watchdog tool to monitor a task and potentially kill it after
|
||||
# not producing any output for $MAX_NO_OUTPUT seconds.
|
||||
#
|
||||
|
||||
# Number of seconds w/o output before printing a stack trace and killing the watched process
|
||||
MAX_NO_OUTPUT=${MAX_NO_OUTPUT:-900}
|
||||
|
||||
# Number of seconds to sleep before checking the output again
|
||||
SLEEP_TIME=${SLEEP_TIME:-20}
|
||||
|
||||
# Internal fields
|
||||
CMD_OUT="/tmp/watchdog.out"
|
||||
CMD_PID="/tmp/watchdog.pid"
|
||||
CMD_EXIT="/tmp/watchdog.exit"
|
||||
|
||||
|
||||
# =============================================
|
||||
# Utility functions
|
||||
# =============================================
|
||||
|
||||
max_of() {
|
||||
local max number
|
||||
|
||||
max="$1"
|
||||
|
||||
for number in "${@:2}"; do
|
||||
if ((number > max)); then
|
||||
max="$number"
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%d\n' "$max"
|
||||
}
|
||||
|
||||
# Returns the highest modification time out of $CMD_OUT (which is the command output file)
|
||||
# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests)
|
||||
mod_time () {
|
||||
CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT`
|
||||
ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null`
|
||||
echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES`
|
||||
}
|
||||
|
||||
the_time() {
|
||||
echo `date +%s`
|
||||
}
|
||||
|
||||
# watchdog process
|
||||
|
||||
watchdog () {
|
||||
touch $CMD_OUT
|
||||
|
||||
while true; do
|
||||
sleep $SLEEP_TIME
|
||||
|
||||
time_diff=$((`the_time` - `mod_time`))
|
||||
|
||||
if [ $time_diff -ge $MAX_NO_OUTPUT ]; then
|
||||
echo "=============================================================================="
|
||||
echo "Process produced no output for ${MAX_NO_OUTPUT} seconds."
|
||||
echo "=============================================================================="
|
||||
|
||||
# run timeout callback
|
||||
$CALLBACK_ON_TIMEOUT
|
||||
|
||||
echo "Killing process with pid=$(<$CMD_PID) and all descendants"
|
||||
pkill -P $(<$CMD_PID) # kill descendants
|
||||
kill $(<$CMD_PID) # kill process itself
|
||||
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# =============================================
|
||||
# main function
|
||||
# =============================================
|
||||
|
||||
# entrypoint
|
||||
function run_with_watchdog() {
|
||||
local cmd="$1"
|
||||
local CALLBACK_ON_TIMEOUT="$2"
|
||||
|
||||
watchdog &
|
||||
WD_PID=$!
|
||||
echo "STARTED watchdog (${WD_PID})."
|
||||
|
||||
echo "RUNNING '${cmd}'."
|
||||
|
||||
# Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to
|
||||
# allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains
|
||||
# the exit code. This is important for CI build life-cycle (success/failure).
|
||||
( $cmd & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT
|
||||
|
||||
EXIT_CODE=$(<$CMD_EXIT)
|
||||
|
||||
echo "Process exited with EXIT CODE: ${EXIT_CODE}."
|
||||
|
||||
# Make sure to kill the watchdog in any case after $CMD has completed
|
||||
echo "Trying to KILL watchdog (${WD_PID})."
|
||||
( kill $WD_PID 2>&1 ) > /dev/null
|
||||
|
||||
rm $CMD_PID
|
||||
rm $CMD_EXIT
|
||||
|
||||
return $EXIT_CODE
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue