[FLINK-35309][cdc] Introduce CI checks for licenses and NOTICE
Co-authored-by: GOODBOY008 <gongzhongqiang@apache.org>pull/3312/head
Qingsheng Ren
9 months ago
parent
e452d66b3b
commit
62cc62ef72
@ -0,0 +1,82 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-cdc-parent</artifactId>
|
||||
<version>${revision}</version>
|
||||
<relativePath>../../..</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>flink-cdc-ci-tools</artifactId>
|
||||
<version>${revision}</version>
|
||||
|
||||
<properties>
|
||||
<japicmp.skip>true</japicmp.skip>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-annotations</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-test-utils-junit</artifactId>
|
||||
<version>${flink.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-slf4j-impl</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<version>${log4j.version}</version>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<version>${log4j.version}</version>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.licensecheck;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/** Utility for checking all things related to License and Notice files. */
|
||||
public class LicenseChecker {
|
||||
// ---------------------------------------- Launcher ---------------------------------------- //
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(LicenseChecker.class);
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
if (args.length < 2) {
|
||||
System.out.println(
|
||||
"Usage: LicenseChecker <pathMavenBuildOutput> <pathFlinkCDCRoot> <pathFlinkCDCDeployed>");
|
||||
System.exit(1);
|
||||
}
|
||||
LOG.warn(
|
||||
"THIS UTILITY IS ONLY CHECKING FOR COMMON LICENSING MISTAKES. A MANUAL CHECK OF THE NOTICE FILES, DEPLOYED ARTIFACTS, ETC. IS STILL NEEDED!");
|
||||
|
||||
int severeIssueCount = NoticeFileChecker.run(new File(args[0]), Paths.get(args[1]));
|
||||
|
||||
severeIssueCount += JarFileChecker.checkPath(Paths.get(args[2]));
|
||||
|
||||
if (severeIssueCount > 0) {
|
||||
LOG.warn("Found a total of {} severe license issues", severeIssueCount);
|
||||
|
||||
System.exit(1);
|
||||
}
|
||||
LOG.info("License check completed without severe issues.");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,392 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.licensecheck;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.deploy.DeployParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.notice.NoticeContents;
|
||||
import org.apache.flink.cdc.tools.ci.utils.notice.NoticeParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shade.ShadeParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Utility class checking for proper NOTICE files based on the maven build output. */
|
||||
public class NoticeFileChecker {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class);
|
||||
|
||||
private static final List<String> MODULES_DEFINING_EXCESS_DEPENDENCIES =
|
||||
loadFromResources("modules-defining-excess-dependencies.modulelist");
|
||||
|
||||
// Examples:
|
||||
// "- org.apache.htrace:htrace-core:3.1.0-incubating"
|
||||
// or
|
||||
// "This project bundles "net.jcip:jcip-annotations:1.0".
|
||||
private static final Pattern NOTICE_DEPENDENCY_PATTERN =
|
||||
Pattern.compile(
|
||||
"- ([^ :]+):([^:]+):([^ ]+)($| )|.*bundles \"([^:]+):([^:]+):([^\"]+)\".*");
|
||||
|
||||
static int run(File buildResult, Path root) throws IOException {
|
||||
// parse included dependencies from build output
|
||||
final Map<String, Set<Dependency>> modulesWithBundledDependencies =
|
||||
combineAndFilterFlinkDependencies(
|
||||
ShadeParser.parseShadeOutput(buildResult.toPath()),
|
||||
DependencyParser.parseDependencyCopyOutput(buildResult.toPath()));
|
||||
|
||||
final Set<String> deployedModules = DeployParser.parseDeployOutput(buildResult);
|
||||
|
||||
LOG.info(
|
||||
"Extracted "
|
||||
+ deployedModules.size()
|
||||
+ " modules that were deployed and "
|
||||
+ modulesWithBundledDependencies.keySet().size()
|
||||
+ " modules which bundle dependencies with a total of "
|
||||
+ modulesWithBundledDependencies.values().size()
|
||||
+ " dependencies");
|
||||
|
||||
// find modules producing a shaded-jar
|
||||
List<Path> noticeFiles = findNoticeFiles(root);
|
||||
LOG.info("Found {} NOTICE files to check", noticeFiles.size());
|
||||
|
||||
final Map<String, Optional<NoticeContents>> moduleToNotice =
|
||||
noticeFiles.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
NoticeFileChecker::getModuleFromNoticeFile,
|
||||
noticeFile -> {
|
||||
try {
|
||||
return NoticeParser.parseNoticeFile(noticeFile);
|
||||
} catch (IOException e) {
|
||||
// some machine issue
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}));
|
||||
|
||||
return run(modulesWithBundledDependencies, deployedModules, moduleToNotice);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static int run(
|
||||
Map<String, Set<Dependency>> modulesWithBundledDependencies,
|
||||
Set<String> deployedModules,
|
||||
Map<String, Optional<NoticeContents>> noticeFiles)
|
||||
throws IOException {
|
||||
int severeIssueCount = 0;
|
||||
|
||||
final Set<String> modulesSkippingDeployment =
|
||||
new HashSet<>(modulesWithBundledDependencies.keySet());
|
||||
modulesSkippingDeployment.removeAll(deployedModules);
|
||||
|
||||
LOG.debug(
|
||||
"The following {} modules are skipping deployment: {}",
|
||||
modulesSkippingDeployment.size(),
|
||||
modulesSkippingDeployment.stream()
|
||||
.sorted()
|
||||
.collect(Collectors.joining("\n\t", "\n\t", "")));
|
||||
|
||||
for (String moduleSkippingDeployment : modulesSkippingDeployment) {
|
||||
// TODO: this doesn't work for modules requiring a NOTICE that are bundled indirectly
|
||||
// TODO: via another non-deployed module
|
||||
boolean bundledByDeployedModule =
|
||||
modulesWithBundledDependencies.entrySet().stream()
|
||||
.filter(
|
||||
entry ->
|
||||
entry.getValue().stream()
|
||||
.map(Dependency::getArtifactId)
|
||||
.anyMatch(
|
||||
artifactId ->
|
||||
artifactId.equals(
|
||||
moduleSkippingDeployment)))
|
||||
.anyMatch(entry -> !modulesSkippingDeployment.contains(entry.getKey()));
|
||||
|
||||
if (!bundledByDeployedModule) {
|
||||
modulesWithBundledDependencies.remove(moduleSkippingDeployment);
|
||||
} else {
|
||||
LOG.debug(
|
||||
"Including module {} in license checks, despite not being deployed, because it is bundled by another deployed module.",
|
||||
moduleSkippingDeployment);
|
||||
}
|
||||
}
|
||||
|
||||
// check that all required NOTICE files exists
|
||||
severeIssueCount +=
|
||||
ensureRequiredNoticeFiles(modulesWithBundledDependencies, noticeFiles.keySet());
|
||||
|
||||
// check each NOTICE file
|
||||
for (Map.Entry<String, Optional<NoticeContents>> noticeFile : noticeFiles.entrySet()) {
|
||||
severeIssueCount +=
|
||||
checkNoticeFileAndLogProblems(
|
||||
modulesWithBundledDependencies,
|
||||
noticeFile.getKey(),
|
||||
noticeFile.getValue().orElse(null));
|
||||
}
|
||||
|
||||
return severeIssueCount;
|
||||
}
|
||||
|
||||
private static Map<String, Set<Dependency>> combineAndFilterFlinkDependencies(
|
||||
Map<String, Set<Dependency>> modulesWithBundledDependencies,
|
||||
Map<String, Set<Dependency>> modulesWithCopiedDependencies) {
|
||||
|
||||
final Map<String, Set<Dependency>> combinedAndFiltered = new LinkedHashMap<>();
|
||||
|
||||
Stream.concat(
|
||||
modulesWithBundledDependencies.entrySet().stream(),
|
||||
modulesWithCopiedDependencies.entrySet().stream())
|
||||
.forEach(
|
||||
(entry) -> {
|
||||
final Set<Dependency> dependencies =
|
||||
combinedAndFiltered.computeIfAbsent(
|
||||
entry.getKey(), ignored -> new LinkedHashSet<>());
|
||||
|
||||
for (Dependency dependency : entry.getValue()) {
|
||||
if (!dependency.getGroupId().contains("org.apache.flink")) {
|
||||
dependencies.add(dependency);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return combinedAndFiltered;
|
||||
}
|
||||
|
||||
private static int ensureRequiredNoticeFiles(
|
||||
Map<String, Set<Dependency>> modulesWithShadedDependencies,
|
||||
Collection<String> modulesWithNoticeFile) {
|
||||
int severeIssueCount = 0;
|
||||
Set<String> shadingModules = new HashSet<>(modulesWithShadedDependencies.keySet());
|
||||
shadingModules.removeAll(modulesWithNoticeFile);
|
||||
for (String moduleWithoutNotice : shadingModules) {
|
||||
if (modulesWithShadedDependencies.get(moduleWithoutNotice).stream()
|
||||
.anyMatch(dependency -> !dependency.getGroupId().equals("org.apache.flink"))) {
|
||||
LOG.error(
|
||||
"Module {} is missing a NOTICE file. It has shaded dependencies: {}",
|
||||
moduleWithoutNotice,
|
||||
modulesWithShadedDependencies.get(moduleWithoutNotice).stream()
|
||||
.map(Dependency::toString)
|
||||
.collect(Collectors.joining("\n\t", "\n\t", "")));
|
||||
severeIssueCount++;
|
||||
}
|
||||
}
|
||||
return severeIssueCount;
|
||||
}
|
||||
|
||||
private static String getModuleFromNoticeFile(Path noticeFile) {
|
||||
Path moduleDirectory =
|
||||
noticeFile
|
||||
.getParent() // META-INF
|
||||
.getParent() // resources
|
||||
.getParent() // main
|
||||
.getParent() // src
|
||||
.getParent(); // <-- module name
|
||||
return moduleDirectory.getFileName().toString();
|
||||
}
|
||||
|
||||
private static int checkNoticeFileAndLogProblems(
|
||||
Map<String, Set<Dependency>> modulesWithShadedDependencies,
|
||||
String moduleName,
|
||||
@Nullable NoticeContents noticeContents)
|
||||
throws IOException {
|
||||
|
||||
final Map<Severity, List<String>> problemsBySeverity =
|
||||
checkNoticeFile(modulesWithShadedDependencies, moduleName, noticeContents);
|
||||
|
||||
final List<String> severeProblems =
|
||||
problemsBySeverity.getOrDefault(Severity.CRITICAL, Collections.emptyList());
|
||||
|
||||
if (!problemsBySeverity.isEmpty()) {
|
||||
final List<String> toleratedProblems =
|
||||
problemsBySeverity.getOrDefault(Severity.TOLERATED, Collections.emptyList());
|
||||
final List<String> expectedProblems =
|
||||
problemsBySeverity.getOrDefault(Severity.SUPPRESSED, Collections.emptyList());
|
||||
|
||||
LOG.info(
|
||||
"Problems were detected for a NOTICE file.\n" + "\t{}:\n" + "{}{}{}",
|
||||
moduleName,
|
||||
convertProblemsToIndentedString(
|
||||
severeProblems,
|
||||
"These issue are legally problematic and MUST be fixed:"),
|
||||
convertProblemsToIndentedString(
|
||||
toleratedProblems,
|
||||
"These issues are mistakes that aren't legally problematic. They SHOULD be fixed at some point, but we don't have to:"),
|
||||
convertProblemsToIndentedString(
|
||||
expectedProblems, "These issues are assumed to be false-positives:"));
|
||||
}
|
||||
|
||||
return severeProblems.size();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Map<Severity, List<String>> checkNoticeFile(
|
||||
Map<String, Set<Dependency>> modulesWithShadedDependencies,
|
||||
String moduleName,
|
||||
@Nullable NoticeContents noticeContents) {
|
||||
|
||||
final Map<Severity, List<String>> problemsBySeverity = new HashMap<>();
|
||||
|
||||
if (noticeContents == null) {
|
||||
addProblem(problemsBySeverity, Severity.CRITICAL, "The NOTICE file was empty.");
|
||||
} else {
|
||||
// first line must be the module name.
|
||||
if (!noticeContents.getNoticeModuleName().equals(moduleName)) {
|
||||
addProblem(
|
||||
problemsBySeverity,
|
||||
Severity.TOLERATED,
|
||||
String.format(
|
||||
"First line does not start with module name. firstLine=%s",
|
||||
noticeContents.getNoticeModuleName()));
|
||||
}
|
||||
|
||||
// collect all declared dependencies from NOTICE file
|
||||
Set<Dependency> declaredDependencies = new HashSet<>();
|
||||
for (Dependency declaredDependency : noticeContents.getDeclaredDependencies()) {
|
||||
if (!declaredDependencies.add(declaredDependency)) {
|
||||
addProblem(
|
||||
problemsBySeverity,
|
||||
Severity.CRITICAL,
|
||||
String.format("Dependency %s is declared twice.", declaredDependency));
|
||||
}
|
||||
}
|
||||
|
||||
// find all dependencies missing from NOTICE file
|
||||
Collection<Dependency> expectedDependencies =
|
||||
modulesWithShadedDependencies.getOrDefault(moduleName, Collections.emptySet())
|
||||
.stream()
|
||||
.filter(
|
||||
dependency ->
|
||||
!dependency.getGroupId().equals("org.apache.flink"))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (Dependency expectedDependency : expectedDependencies) {
|
||||
if (!declaredDependencies.contains(expectedDependency)) {
|
||||
addProblem(
|
||||
problemsBySeverity,
|
||||
Severity.CRITICAL,
|
||||
String.format("Dependency %s is not listed.", expectedDependency));
|
||||
}
|
||||
}
|
||||
|
||||
boolean moduleDefinesExcessDependencies =
|
||||
MODULES_DEFINING_EXCESS_DEPENDENCIES.contains(moduleName);
|
||||
|
||||
// find all dependencies defined in NOTICE file, which were not expected
|
||||
for (Dependency declaredDependency : declaredDependencies) {
|
||||
if (!expectedDependencies.contains(declaredDependency)) {
|
||||
final Severity severity =
|
||||
moduleDefinesExcessDependencies
|
||||
? Severity.SUPPRESSED
|
||||
: Severity.TOLERATED;
|
||||
addProblem(
|
||||
problemsBySeverity,
|
||||
severity,
|
||||
String.format(
|
||||
"Dependency %s is not bundled, but listed.",
|
||||
declaredDependency));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return problemsBySeverity;
|
||||
}
|
||||
|
||||
private static void addProblem(
|
||||
Map<Severity, List<String>> problemsBySeverity, Severity severity, String problem) {
|
||||
problemsBySeverity.computeIfAbsent(severity, ignored -> new ArrayList<>()).add(problem);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
enum Severity {
|
||||
/** Issues that a legally problematic which must be fixed. */
|
||||
CRITICAL,
|
||||
/** Issues that affect the correctness but aren't legally problematic. */
|
||||
TOLERATED,
|
||||
/** Issues where we intentionally break the rules. */
|
||||
SUPPRESSED
|
||||
}
|
||||
|
||||
private static String convertProblemsToIndentedString(List<String> problems, String header) {
|
||||
return problems.isEmpty()
|
||||
? ""
|
||||
: problems.stream()
|
||||
.map(s -> "\t\t\t" + s)
|
||||
.collect(Collectors.joining("\n", "\t\t " + header + " \n", "\n"));
|
||||
}
|
||||
|
||||
private static List<Path> findNoticeFiles(Path root) throws IOException {
|
||||
return Files.walk(root)
|
||||
.filter(
|
||||
file -> {
|
||||
int nameCount = file.getNameCount();
|
||||
return nameCount >= 3
|
||||
&& file.getName(nameCount - 3).toString().equals("resources")
|
||||
&& file.getName(nameCount - 2).toString().equals("META-INF")
|
||||
&& file.getName(nameCount - 1).toString().equals("NOTICE");
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private static List<String> loadFromResources(String fileName) {
|
||||
try {
|
||||
try (BufferedReader bufferedReader =
|
||||
new BufferedReader(
|
||||
new InputStreamReader(
|
||||
Objects.requireNonNull(
|
||||
NoticeFileChecker.class.getResourceAsStream(
|
||||
"/" + fileName))))) {
|
||||
|
||||
List<String> result =
|
||||
bufferedReader
|
||||
.lines()
|
||||
.filter(line -> !line.startsWith("#") && !line.isEmpty())
|
||||
.collect(Collectors.toList());
|
||||
LOG.debug("Loaded {} items from resource {}", result.size(), fileName);
|
||||
return result;
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
// wrap anything in a RuntimeException to be callable from the static initializer
|
||||
throw new RuntimeException("Error while loading resource", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,266 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.optional;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shade.ShadeParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Verifies that all dependencies bundled with the shade-plugin are marked as optional in the pom.
|
||||
* This ensures compatibility with later maven versions and in general simplifies dependency
|
||||
* management as transitivity is no longer dependent on the shade-plugin.
|
||||
*
|
||||
* <p>In Maven 3.3 the dependency tree was made immutable at runtime, and thus can no longer be
|
||||
* changed by the shade plugin. The plugin would usually remove a dependency from the tree when it
|
||||
* is being bundled (known as dependency reduction). While dependency reduction still works for the
|
||||
* published poms (== what users consume) since it can still change the content of the final pom,
|
||||
* while developing Flink it no longer works. This breaks plenty of things, since suddenly a bunch
|
||||
* of dependencies are still visible to downstream modules that weren't before.
|
||||
*
|
||||
* <p>To workaround this we mark all dependencies that we bundle as optional; this makes them
|
||||
* non-transitive. To a downstream module, behavior-wise a non-transitive dependency is identical to
|
||||
* a removed dependency.
|
||||
*
|
||||
* <p>This checker analyzes the bundled dependencies (based on the shade-plugin output) and the set
|
||||
* of dependencies (based on the dependency plugin) to detect cases where a dependency is not marked
|
||||
* as optional as it should.
|
||||
*
|
||||
* <p>The enforced rule is rather simple: Any dependency that is bundled, or any of its parents,
|
||||
* must show up as optional in the dependency tree. The parent clause is required to cover cases
|
||||
* where a module has 2 paths to a bundled dependency. If a module depends on A1/A2, each depending
|
||||
* on B, with A1 and B being bundled, then even if A1 is marked as optional B is still shown as a
|
||||
* non-optional dependency (because the non-optional A2 still needs it!).
|
||||
*/
|
||||
public class ShadeOptionalChecker {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ShadeOptionalChecker.class);
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
if (args.length < 2) {
|
||||
System.out.println(
|
||||
"Usage: ShadeOptionalChecker <pathShadeBuildOutput> <pathMavenDependencyOutput>");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
final Path shadeOutputPath = Paths.get(args[0]);
|
||||
final Path dependencyOutputPath = Paths.get(args[1]);
|
||||
|
||||
final Map<String, Set<Dependency>> bundledDependenciesByModule =
|
||||
ShadeParser.parseShadeOutput(shadeOutputPath);
|
||||
final Map<String, DependencyTree> dependenciesByModule =
|
||||
DependencyParser.parseDependencyTreeOutput(dependencyOutputPath);
|
||||
|
||||
final Map<String, Set<Dependency>> violations =
|
||||
checkOptionalFlags(bundledDependenciesByModule, dependenciesByModule);
|
||||
|
||||
if (!violations.isEmpty()) {
|
||||
LOG.error(
|
||||
"{} modules bundle in total {} dependencies without them being marked as optional in the pom.",
|
||||
violations.keySet().size(),
|
||||
violations.values().stream().mapToInt(Collection::size).sum());
|
||||
LOG.error(
|
||||
"\tIn order for shading to properly work within Flink we require all bundled dependencies to be marked as optional in the pom.");
|
||||
LOG.error(
|
||||
"\tFor verification purposes we require the dependency tree from the dependency-plugin to show the dependency as either:");
|
||||
LOG.error("\t\ta) an optional dependency,");
|
||||
LOG.error("\t\tb) a transitive dependency of another optional dependency.");
|
||||
LOG.error(
|
||||
"\tIn most cases adding '<optional>${flink.markBundledAsOptional}</optional>' to the bundled dependency is sufficient.");
|
||||
LOG.error(
|
||||
"\tThere are some edge cases where a transitive dependency might be associated with the \"wrong\" dependency in the tree, for example if a test dependency also requires it.");
|
||||
LOG.error(
|
||||
"\tIn such cases you need to adjust the poms so that the dependency shows up in the right spot. This may require adding an explicit dependency (Management) entry, excluding dependencies, or at times even reordering dependencies in the pom.");
|
||||
LOG.error(
|
||||
"\tSee the Dependencies page in the wiki for details: https://cwiki.apache.org/confluence/display/FLINK/Dependencies");
|
||||
|
||||
for (String moduleWithViolations : violations.keySet()) {
|
||||
final Collection<Dependency> dependencyViolations =
|
||||
violations.get(moduleWithViolations);
|
||||
LOG.error(
|
||||
"\tModule {} ({} violation{}):",
|
||||
moduleWithViolations,
|
||||
dependencyViolations.size(),
|
||||
dependencyViolations.size() == 1 ? "" : "s");
|
||||
for (Dependency dependencyViolation : dependencyViolations) {
|
||||
LOG.error("\t\t{}", dependencyViolation);
|
||||
}
|
||||
}
|
||||
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, Set<Dependency>> checkOptionalFlags(
|
||||
Map<String, Set<Dependency>> bundledDependenciesByModule,
|
||||
Map<String, DependencyTree> dependenciesByModule) {
|
||||
|
||||
final Map<String, Set<Dependency>> allViolations = new HashMap<>();
|
||||
|
||||
for (String module : bundledDependenciesByModule.keySet()) {
|
||||
LOG.debug("Checking module '{}'.", module);
|
||||
if (!dependenciesByModule.containsKey(module)) {
|
||||
throw new IllegalStateException(
|
||||
String.format(
|
||||
"Module %s listed by shade-plugin, but not dependency-plugin.",
|
||||
module));
|
||||
}
|
||||
|
||||
final Collection<Dependency> bundledDependencies =
|
||||
bundledDependenciesByModule.get(module);
|
||||
final DependencyTree dependencyTree = dependenciesByModule.get(module);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
checkOptionalFlags(module, bundledDependencies, dependencyTree);
|
||||
|
||||
if (violations.isEmpty()) {
|
||||
LOG.info("OK: {}", module);
|
||||
} else {
|
||||
allViolations.put(module, violations);
|
||||
}
|
||||
}
|
||||
|
||||
return allViolations;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Set<Dependency> checkOptionalFlags(
|
||||
String module,
|
||||
Collection<Dependency> bundledDependencies,
|
||||
DependencyTree dependencyTree) {
|
||||
|
||||
bundledDependencies =
|
||||
bundledDependencies.stream()
|
||||
// force-shading isn't relevant for this check but breaks some shortcuts
|
||||
.filter(
|
||||
dependency ->
|
||||
!dependency
|
||||
.getArtifactId()
|
||||
.equals("flink-shaded-force-shading"))
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
final Set<Dependency> violations = new HashSet<>();
|
||||
|
||||
if (bundledDependencies.isEmpty()) {
|
||||
LOG.debug("\tModule is not bundling any dependencies.");
|
||||
return violations;
|
||||
}
|
||||
|
||||
// The set of dependencies that the module directly depends on and which downstream modules
|
||||
// would pull in transitively.
|
||||
//
|
||||
// If this set is empty we do not need to check anything.
|
||||
// This allows us to avoid some edge-cases:
|
||||
//
|
||||
// Assume module M has the following (full) dependency tree, bundling dependency 1 and 2:
|
||||
//
|
||||
// +- dependency1 (compile/optional)",
|
||||
// | \- dependency2 (compile) (implicitly optional because dependency1 is optional)
|
||||
// \- dependency3 (test)
|
||||
// \- dependency2 (compile)
|
||||
//
|
||||
// However, in the dependency plugin output a dependency can only show up once, so Maven may
|
||||
// return this:
|
||||
//
|
||||
// +- dependency1 (compile/optional)",
|
||||
// \- dependency3 (test)
|
||||
// \- dependency2 (compile)
|
||||
//
|
||||
// Given this tree, and knowing that dependency2 is bundled, we would draw the conclusion
|
||||
// that dependency2 is missing the optional flag.
|
||||
//
|
||||
// However, because dependency 1/3 are optional/test dependencies they are not transitive.
|
||||
// Without any direct transitive dependency nothing can leak through to downstream modules,
|
||||
// removing the need to check dependency 2 at all (and in turn, saving us from having to
|
||||
// resolve this problem).
|
||||
final List<Dependency> directTransitiveDependencies =
|
||||
dependencyTree.getDirectDependencies().stream()
|
||||
.filter(
|
||||
dependency ->
|
||||
!(isOptional(dependency)
|
||||
|| hasProvidedScope(dependency)
|
||||
|| hasTestScope(dependency)
|
||||
|| isCommonCompileDependency(dependency)))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// if nothing would be exposed to downstream modules we exit early to reduce noise on CI
|
||||
if (directTransitiveDependencies.isEmpty()) {
|
||||
LOG.debug(
|
||||
"Skipping deep-check of module {} because all direct dependencies are not transitive.",
|
||||
module);
|
||||
return violations;
|
||||
}
|
||||
LOG.debug(
|
||||
"Running deep-check of module {} because there are direct dependencies that are transitive: {}",
|
||||
module,
|
||||
directTransitiveDependencies);
|
||||
|
||||
for (Dependency bundledDependency : bundledDependencies) {
|
||||
LOG.debug("\tChecking dependency '{}'.", bundledDependency);
|
||||
|
||||
final List<Dependency> dependencyPath = dependencyTree.getPathTo(bundledDependency);
|
||||
|
||||
final boolean isOptional =
|
||||
dependencyPath.stream().anyMatch(parent -> parent.isOptional().orElse(false));
|
||||
|
||||
if (!isOptional) {
|
||||
violations.add(bundledDependency);
|
||||
}
|
||||
}
|
||||
|
||||
return violations;
|
||||
}
|
||||
|
||||
private static boolean isOptional(Dependency dependency) {
|
||||
return dependency.isOptional().orElse(false);
|
||||
}
|
||||
|
||||
private static boolean hasProvidedScope(Dependency dependency) {
|
||||
return "provided".equals(dependency.getScope().orElse(null));
|
||||
}
|
||||
|
||||
private static boolean hasTestScope(Dependency dependency) {
|
||||
return "test".equals(dependency.getScope().orElse(null));
|
||||
}
|
||||
|
||||
/**
|
||||
* These are compile dependencies that are set up in the root pom. We do not require modules to
|
||||
* mark these as optional because all modules depend on them anyway; whether they leak through
|
||||
* or not is therefore irrelevant.
|
||||
*/
|
||||
private static boolean isCommonCompileDependency(Dependency dependency) {
|
||||
return "flink-shaded-force-shading".equals(dependency.getArtifactId())
|
||||
|| "jsr305".equals(dependency.getArtifactId())
|
||||
|| "slf4j-api".equals(dependency.getArtifactId());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.suffixcheck;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Utility for checking the presence/absence of scala-suffixes. */
|
||||
public class ScalaSuffixChecker {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ScalaSuffixChecker.class);
|
||||
|
||||
// [INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @ flink-annotations ---
|
||||
private static final Pattern moduleNamePattern =
|
||||
Pattern.compile(".* --- maven-dependency-plugin.* @ (.*) ---.*");
|
||||
|
||||
// [INFO] +- junit:junit:jar:4.13.2:test
|
||||
// [INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test
|
||||
// [INFO] \- org.apache.logging.log4j:log4j-1.2-api:jar:2.14.1:test
|
||||
private static final Pattern blockPattern = Pattern.compile(".* [+|\\\\].*");
|
||||
|
||||
// [INFO] +- org.scala-lang:scala-reflect:jar:2.11.12:test
|
||||
private static final Pattern scalaSuffixPattern = Pattern.compile("_2.1[0-9]");
|
||||
|
||||
private static final Set<String> EXCLUDED_MODULES =
|
||||
new HashSet<>(
|
||||
Arrays.asList(
|
||||
// we ignore flink-rpc-akka because it is loaded through a separate
|
||||
// class loader
|
||||
"flink-rpc-akka",
|
||||
// we ignore flink-table-planner-loader because it loads the planner
|
||||
// through a different classpath
|
||||
"flink-table-planner-loader"));
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
if (args.length < 2) {
|
||||
System.out.println("Usage: ScalaSuffixChecker <pathMavenBuildOutput> <pathFlinkRoot>");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
final Path mavenOutputPath = Paths.get(args[0]);
|
||||
final Path flinkRootPath = Paths.get(args[1]);
|
||||
|
||||
final ParseResult parseResult = parseMavenOutput(mavenOutputPath);
|
||||
if (parseResult.getCleanModules().isEmpty()) {
|
||||
LOG.error("Parsing found 0 scala-free modules; the parsing is likely broken.");
|
||||
System.exit(1);
|
||||
}
|
||||
if (parseResult.getInfectedModules().isEmpty()) {
|
||||
LOG.error("Parsing found 0 scala-dependent modules; the parsing is likely broken.");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
final Collection<String> violations = checkScalaSuffixes(parseResult, flinkRootPath);
|
||||
|
||||
if (!violations.isEmpty()) {
|
||||
LOG.error(
|
||||
"Violations found:{}",
|
||||
violations.stream().collect(Collectors.joining("\n\t", "\n\t", "")));
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static ParseResult parseMavenOutput(final Path path) throws IOException {
|
||||
final Set<String> cleanModules = new HashSet<>();
|
||||
final Set<String> infectedModules = new HashSet<>();
|
||||
|
||||
final Map<String, DependencyTree> dependenciesByModule =
|
||||
DependencyParser.parseDependencyTreeOutput(path);
|
||||
|
||||
for (String module : dependenciesByModule.keySet()) {
|
||||
final String moduleName = stripScalaSuffix(module);
|
||||
if (isExcluded(moduleName)) {
|
||||
continue;
|
||||
}
|
||||
LOG.trace("Processing module '{}'.", moduleName);
|
||||
|
||||
final List<Dependency> dependencies =
|
||||
dependenciesByModule.get(module).flatten().collect(Collectors.toList());
|
||||
|
||||
boolean infected = false;
|
||||
for (Dependency dependency : dependencies) {
|
||||
final boolean dependsOnScala = dependsOnScala(dependency);
|
||||
final boolean isTestDependency = dependency.getScope().get().equals("test");
|
||||
// we ignored flink-rpc-akka because it is loaded through a separate class loader
|
||||
final boolean isExcluded = isExcluded(dependency.getArtifactId());
|
||||
LOG.trace("\tdependency:{}", dependency);
|
||||
LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala);
|
||||
LOG.trace("\t\tis-test-dependency:{}", isTestDependency);
|
||||
LOG.trace("\t\tis-excluded:{}", isExcluded);
|
||||
if (dependsOnScala && !isTestDependency && !isExcluded) {
|
||||
LOG.trace("\t\tOutbreak detected at {}!", moduleName);
|
||||
infected = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (infected) {
|
||||
infectedModules.add(moduleName);
|
||||
} else {
|
||||
cleanModules.add(moduleName);
|
||||
}
|
||||
}
|
||||
|
||||
return new ParseResult(cleanModules, infectedModules);
|
||||
}
|
||||
|
||||
private static String stripScalaSuffix(final String moduleName) {
|
||||
final int i = moduleName.indexOf("_2.");
|
||||
return i > 0 ? moduleName.substring(0, i) : moduleName;
|
||||
}
|
||||
|
||||
private static boolean dependsOnScala(final Dependency dependency) {
|
||||
return dependency.getGroupId().contains("org.scala-lang")
|
||||
|| scalaSuffixPattern.matcher(dependency.getArtifactId()).find();
|
||||
}
|
||||
|
||||
private static Collection<String> checkScalaSuffixes(
|
||||
final ParseResult parseResult, Path flinkRootPath) throws IOException {
|
||||
final Collection<String> violations = new ArrayList<>();
|
||||
|
||||
// exclude e2e modules and flink-docs for convenience as they
|
||||
// a) are not deployed during a release
|
||||
// b) exist only for dev purposes
|
||||
// c) no-one should depend on them
|
||||
final Collection<String> excludedModules = new ArrayList<>();
|
||||
excludedModules.add("flink-docs");
|
||||
excludedModules.addAll(getEndToEndTestModules(flinkRootPath));
|
||||
|
||||
for (String excludedModule : excludedModules) {
|
||||
parseResult.getCleanModules().remove(excludedModule);
|
||||
parseResult.getInfectedModules().remove(excludedModule);
|
||||
}
|
||||
|
||||
violations.addAll(checkCleanModules(parseResult.getCleanModules(), flinkRootPath));
|
||||
violations.addAll(checkInfectedModules(parseResult.getInfectedModules(), flinkRootPath));
|
||||
|
||||
return violations;
|
||||
}
|
||||
|
||||
private static Collection<String> getEndToEndTestModules(Path flinkRootPath)
|
||||
throws IOException {
|
||||
try (Stream<Path> pathStream =
|
||||
Files.walk(flinkRootPath.resolve("flink-end-to-end-tests"), 5)) {
|
||||
return pathStream
|
||||
.filter(path -> path.getFileName().toString().equals("pom.xml"))
|
||||
.map(path -> path.getParent().getFileName().toString())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
private static Collection<String> checkCleanModules(
|
||||
Collection<String> modules, Path flinkRootPath) throws IOException {
|
||||
return checkModules(
|
||||
modules,
|
||||
flinkRootPath,
|
||||
"_${scala.binary.version}",
|
||||
"Scala-free module '%s' is referenced with scala suffix in '%s'.");
|
||||
}
|
||||
|
||||
private static Collection<String> checkInfectedModules(
|
||||
Collection<String> modules, Path flinkRootPath) throws IOException {
|
||||
return checkModules(
|
||||
modules,
|
||||
flinkRootPath,
|
||||
"",
|
||||
"Scala-dependent module '%s' is referenced without scala suffix in '%s'.");
|
||||
}
|
||||
|
||||
private static Collection<String> checkModules(
|
||||
Collection<String> modules,
|
||||
Path flinkRootPath,
|
||||
String moduleSuffix,
|
||||
String violationTemplate)
|
||||
throws IOException {
|
||||
|
||||
final ArrayList<String> sortedModules = new ArrayList<>(modules);
|
||||
sortedModules.sort(String::compareTo);
|
||||
|
||||
final Collection<String> violations = new ArrayList<>();
|
||||
for (String module : sortedModules) {
|
||||
int numPreviousViolations = violations.size();
|
||||
try (Stream<Path> pathStream = Files.walk(flinkRootPath, 3)) {
|
||||
final List<Path> pomFiles =
|
||||
pathStream
|
||||
.filter(path -> path.getFileName().toString().equals("pom.xml"))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (Path pomFile : pomFiles) {
|
||||
try (Stream<String> lines = Files.lines(pomFile, StandardCharsets.UTF_8)) {
|
||||
final boolean existsCleanReference =
|
||||
lines.anyMatch(
|
||||
line ->
|
||||
line.contains(
|
||||
module + moduleSuffix + "</artifactId>"));
|
||||
|
||||
if (existsCleanReference) {
|
||||
violations.add(
|
||||
String.format(
|
||||
violationTemplate,
|
||||
module,
|
||||
flinkRootPath.relativize(pomFile)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (numPreviousViolations == violations.size()) {
|
||||
LOG.info("OK {}", module);
|
||||
}
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
|
||||
private static boolean isExcluded(String line) {
|
||||
return EXCLUDED_MODULES.stream().anyMatch(line::contains);
|
||||
}
|
||||
|
||||
private static class ParseResult {
|
||||
|
||||
private final Set<String> cleanModules;
|
||||
private final Set<String> infectedModules;
|
||||
|
||||
private ParseResult(Set<String> cleanModules, Set<String> infectedModules) {
|
||||
this.cleanModules = cleanModules;
|
||||
this.infectedModules = infectedModules;
|
||||
}
|
||||
|
||||
public Set<String> getCleanModules() {
|
||||
return cleanModules;
|
||||
}
|
||||
|
||||
public Set<String> getInfectedModules() {
|
||||
return infectedModules;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,228 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.dependency;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.Stack;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Parsing utils for the Maven dependency plugin. */
|
||||
public class DependencyParser {
|
||||
|
||||
private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN =
|
||||
Pattern.compile(
|
||||
".*maven-dependency-plugin:[^:]+:copy .* @ (?<module>[^ _]+)(?:_[0-9.]+)? --.*");
|
||||
|
||||
private static final Pattern DEPENDENCY_TREE_NEXT_MODULE_PATTERN =
|
||||
Pattern.compile(
|
||||
".*maven-dependency-plugin:[^:]+:tree .* @ (?<module>[^ _]+)(?:_[0-9.]+)? --.*");
|
||||
|
||||
/** See {@link DependencyParserTreeTest} for examples. */
|
||||
private static final Pattern DEPENDENCY_TREE_ITEM_PATTERN =
|
||||
Pattern.compile(
|
||||
".* +"
|
||||
+ "(?<groupId>.*?):"
|
||||
+ "(?<artifactId>.*?):"
|
||||
+ "(?<type>.*?):"
|
||||
+ "(?:(?<classifier>.*?):)?"
|
||||
+ "(?<version>.*?):"
|
||||
+ "(?<scope>[^ ]*)"
|
||||
+ "(?<optional> \\(optional\\))?");
|
||||
|
||||
/** See {@link DependencyParserCopyTest} for examples. */
|
||||
private static final Pattern DEPENDENCY_COPY_ITEM_PATTERN =
|
||||
Pattern.compile(
|
||||
".* Configured Artifact: +"
|
||||
+ "(?<groupId>.*?):"
|
||||
+ "(?<artifactId>.*?):"
|
||||
+ "(?:(?<classifier>.*?):)?"
|
||||
+ "(?<version>.*?):"
|
||||
+ "(?:\\?:)?" // unknown cause; e.g.: javax.xml.bind:jaxb-api:?:jar
|
||||
+ "(?<type>.*)");
|
||||
|
||||
/**
|
||||
* Parses the output of a Maven build where {@code dependency:copy} was used, and returns a set
|
||||
* of copied dependencies for each module.
|
||||
*
|
||||
* <p>The returned dependencies will NEVER contain the scope or optional flag.
|
||||
*/
|
||||
public static Map<String, Set<Dependency>> parseDependencyCopyOutput(Path buildOutput)
|
||||
throws IOException {
|
||||
return processLines(buildOutput, DependencyParser::parseDependencyCopyOutput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the output of a Maven build where {@code dependency:tree} was used, and returns a set
|
||||
* of dependencies for each module.
|
||||
*/
|
||||
public static Map<String, DependencyTree> parseDependencyTreeOutput(Path buildOutput)
|
||||
throws IOException {
|
||||
return processLines(buildOutput, DependencyParser::parseDependencyTreeOutput);
|
||||
}
|
||||
|
||||
private static <X> X processLines(Path buildOutput, Function<Stream<String>, X> processor)
|
||||
throws IOException {
|
||||
try (Stream<String> lines = Files.lines(buildOutput)) {
|
||||
return processor.apply(lines.filter(line -> line.contains("[INFO]")));
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Map<String, Set<Dependency>> parseDependencyCopyOutput(Stream<String> lines) {
|
||||
return ParserUtils.parsePluginOutput(
|
||||
lines,
|
||||
DEPENDENCY_COPY_NEXT_MODULE_PATTERN,
|
||||
DependencyParser::parseCopyDependencyBlock);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Map<String, DependencyTree> parseDependencyTreeOutput(Stream<String> lines) {
|
||||
return ParserUtils.parsePluginOutput(
|
||||
lines,
|
||||
DEPENDENCY_TREE_NEXT_MODULE_PATTERN,
|
||||
DependencyParser::parseTreeDependencyBlock);
|
||||
}
|
||||
|
||||
private static Set<Dependency> parseCopyDependencyBlock(Iterator<String> block) {
|
||||
final Set<Dependency> dependencies = new LinkedHashSet<>();
|
||||
|
||||
Optional<Dependency> parsedDependency = parseCopyDependency(block.next());
|
||||
while (parsedDependency.isPresent()) {
|
||||
dependencies.add(parsedDependency.get());
|
||||
|
||||
if (block.hasNext()) {
|
||||
parsedDependency = parseCopyDependency(block.next());
|
||||
} else {
|
||||
parsedDependency = Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
private static DependencyTree parseTreeDependencyBlock(Iterator<String> block) {
|
||||
// discard one line, which only contains the current module name
|
||||
block.next();
|
||||
|
||||
if (!block.hasNext()) {
|
||||
throw new IllegalStateException("Expected more output from the dependency-plugin.");
|
||||
}
|
||||
|
||||
final DependencyTree dependencies = new DependencyTree();
|
||||
|
||||
final Stack<Dependency> parentStack = new Stack<>();
|
||||
final Stack<Integer> treeDepthStack = new Stack<>();
|
||||
String line = block.next();
|
||||
Optional<Dependency> parsedDependency = parseTreeDependency(line);
|
||||
while (parsedDependency.isPresent()) {
|
||||
int treeDepth = getDepth(line);
|
||||
|
||||
while (!treeDepthStack.isEmpty() && treeDepth <= treeDepthStack.peek()) {
|
||||
parentStack.pop();
|
||||
treeDepthStack.pop();
|
||||
}
|
||||
|
||||
final Dependency dependency = parsedDependency.get();
|
||||
|
||||
if (parentStack.isEmpty()) {
|
||||
dependencies.addDirectDependency(dependency);
|
||||
} else {
|
||||
dependencies.addTransitiveDependencyTo(dependency, parentStack.peek());
|
||||
}
|
||||
|
||||
if (treeDepthStack.isEmpty() || treeDepth > treeDepthStack.peek()) {
|
||||
treeDepthStack.push(treeDepth);
|
||||
parentStack.push(dependency);
|
||||
}
|
||||
|
||||
if (block.hasNext()) {
|
||||
line = block.next();
|
||||
parsedDependency = parseTreeDependency(line);
|
||||
} else {
|
||||
parsedDependency = Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Optional<Dependency> parseCopyDependency(String line) {
|
||||
Matcher dependencyMatcher = DEPENDENCY_COPY_ITEM_PATTERN.matcher(line);
|
||||
if (!dependencyMatcher.find()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(
|
||||
Dependency.create(
|
||||
dependencyMatcher.group("groupId"),
|
||||
dependencyMatcher.group("artifactId"),
|
||||
dependencyMatcher.group("version"),
|
||||
dependencyMatcher.group("classifier")));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Optional<Dependency> parseTreeDependency(String line) {
|
||||
Matcher dependencyMatcher = DEPENDENCY_TREE_ITEM_PATTERN.matcher(line);
|
||||
if (!dependencyMatcher.find()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(
|
||||
Dependency.create(
|
||||
dependencyMatcher.group("groupId"),
|
||||
dependencyMatcher.group("artifactId"),
|
||||
dependencyMatcher.group("version"),
|
||||
dependencyMatcher.group("classifier"),
|
||||
dependencyMatcher.group("scope"),
|
||||
dependencyMatcher.group("optional") != null));
|
||||
}
|
||||
|
||||
/**
|
||||
* The depths returned by this method do NOT return a continuous sequence.
|
||||
*
|
||||
* <pre>
|
||||
* +- org.apache.flink:...
|
||||
* | +- org.apache.flink:...
|
||||
* | | \- org.apache.flink:...
|
||||
* ...
|
||||
* </pre>
|
||||
*/
|
||||
private static int getDepth(String line) {
|
||||
final int level = line.indexOf('+');
|
||||
if (level != -1) {
|
||||
return level;
|
||||
}
|
||||
return line.indexOf('\\');
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.deploy;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Parsing utils for the Maven deploy plugin. */
|
||||
public class DeployParser {
|
||||
|
||||
// Examples:
|
||||
//
|
||||
// Deployment on CI with alternative repo
|
||||
// [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---
|
||||
// [INFO] Using alternate deployment repository.../tmp/flink-validation-deployment
|
||||
//
|
||||
// Skipped deployment:
|
||||
// [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---
|
||||
// [INFO] Skipping artifact deployment
|
||||
private static final Pattern DEPLOY_MODULE_PATTERN =
|
||||
Pattern.compile(
|
||||
".maven-deploy-plugin:.*:deploy .* @ (?<module>[^ _]+)(?:_[0-9.]+)? --.*");
|
||||
|
||||
/**
|
||||
* Parses the output of a Maven build where {@code deploy:deploy} was used, and returns a set of
|
||||
* deployed modules.
|
||||
*/
|
||||
public static Set<String> parseDeployOutput(File buildResult) throws IOException {
|
||||
try (Stream<String> linesStream = Files.lines(buildResult.toPath())) {
|
||||
return parseDeployOutput(linesStream);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Set<String> parseDeployOutput(Stream<String> lines) {
|
||||
return ParserUtils.parsePluginOutput(
|
||||
lines, DEPLOY_MODULE_PATTERN, DeployParser::parseDeployBlock)
|
||||
.entrySet().stream()
|
||||
.filter(Map.Entry::getValue)
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private static boolean parseDeployBlock(Iterator<String> block) {
|
||||
return block.hasNext() && !block.next().contains("Skipping artifact deployment");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.notice;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
/** Represents the parsed contents of a NOTICE file. */
|
||||
public class NoticeContents {
|
||||
private final String noticeModuleName;
|
||||
private final Collection<Dependency> declaredDependencies;
|
||||
|
||||
public NoticeContents(String noticeModuleName, Collection<Dependency> declaredDependencies) {
|
||||
this.noticeModuleName = noticeModuleName;
|
||||
this.declaredDependencies = declaredDependencies;
|
||||
}
|
||||
|
||||
public String getNoticeModuleName() {
|
||||
return noticeModuleName;
|
||||
}
|
||||
|
||||
public Collection<Dependency> getDeclaredDependencies() {
|
||||
return declaredDependencies;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.notice;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Parsing utils for NOTICE files. */
|
||||
public class NoticeParser {
|
||||
|
||||
// "- org.apache.htrace:htrace-core:3.1.0-incubating"
|
||||
private static final Pattern NOTICE_DEPENDENCY_PATTERN =
|
||||
Pattern.compile(
|
||||
"- "
|
||||
+ "(?<groupId>[^ ]*?):"
|
||||
+ "(?<artifactId>[^ ]*?):"
|
||||
+ "(?:(?<classifier>[^ ]*?):)?"
|
||||
+ "(?<version>[^ ]*?)"
|
||||
+ "($| )");
|
||||
// "This project bundles "net.jcip:jcip-annotations:1.0".
|
||||
private static final Pattern NOTICE_BUNDLES_DEPENDENCY_PATTERN =
|
||||
Pattern.compile(
|
||||
".*bundles \""
|
||||
+ "(?<groupId>[^ ]*?):"
|
||||
+ "(?<artifactId>[^ ]*?):"
|
||||
+ "(?:(?<classifier>[^ ]*?):)?"
|
||||
+ "(?<version>[^ ]*?)"
|
||||
+ "\".*");
|
||||
|
||||
public static Optional<NoticeContents> parseNoticeFile(Path noticeFile) throws IOException {
|
||||
// 1st line contains module name
|
||||
final List<String> noticeContents = Files.readAllLines(noticeFile);
|
||||
|
||||
return parseNoticeFile(noticeContents);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Optional<NoticeContents> parseNoticeFile(List<String> noticeContents) {
|
||||
if (noticeContents.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
final String noticeModuleName = noticeContents.get(0);
|
||||
|
||||
Collection<Dependency> declaredDependencies = new ArrayList<>();
|
||||
for (String line : noticeContents) {
|
||||
Optional<Dependency> dependency = tryParsing(NOTICE_DEPENDENCY_PATTERN, line);
|
||||
if (!dependency.isPresent()) {
|
||||
dependency = tryParsing(NOTICE_BUNDLES_DEPENDENCY_PATTERN, line);
|
||||
}
|
||||
dependency.ifPresent(declaredDependencies::add);
|
||||
}
|
||||
|
||||
return Optional.of(new NoticeContents(noticeModuleName, declaredDependencies));
|
||||
}
|
||||
|
||||
private static Optional<Dependency> tryParsing(Pattern pattern, String line) {
|
||||
Matcher matcher = pattern.matcher(line);
|
||||
if (matcher.find()) {
|
||||
String groupId = matcher.group("groupId");
|
||||
String artifactId = matcher.group("artifactId");
|
||||
String version = matcher.group("version");
|
||||
String classifier = matcher.group("classifier");
|
||||
return Optional.of(Dependency.create(groupId, artifactId, version, classifier));
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shade;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Utils for parsing the shade-plugin output. */
|
||||
public final class ShadeParser {
|
||||
|
||||
private static final Pattern SHADE_NEXT_MODULE_PATTERN =
|
||||
Pattern.compile(
|
||||
".*:shade \\((?:shade-flink|shade-dist|default)\\) @ (?<module>[^ _]+)(?:_[0-9.]+)? --.*");
|
||||
|
||||
private static final Pattern SHADE_INCLUDE_MODULE_PATTERN =
|
||||
Pattern.compile(
|
||||
".* "
|
||||
+ "(?<groupId>.*?):"
|
||||
+ "(?<artifactId>.*?):"
|
||||
+ "(?<type>.*?):"
|
||||
+ "(?:(?<classifier>.*?):)?"
|
||||
+ "(?<version>.*?)"
|
||||
+ " in the shaded jar");
|
||||
|
||||
/**
|
||||
* Parses the output of a Maven build where {@code shade:shade} was used, and returns a set of
|
||||
* bundled dependencies for each module.
|
||||
*
|
||||
* <p>The returned dependencies will NEVER contain the scope or optional flag.
|
||||
*
|
||||
* <p>This method only considers the {@code shade-flink} and {@code shade-dist} executions,
|
||||
* because all artifacts we produce that are either published or referenced are created by these
|
||||
* executions. In other words, all artifacts from other executions are only used internally by
|
||||
* the module that created them.
|
||||
*/
|
||||
public static Map<String, Set<Dependency>> parseShadeOutput(Path buildOutput)
|
||||
throws IOException {
|
||||
try (Stream<String> lines = Files.lines(buildOutput)) {
|
||||
return parseShadeOutput(lines);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Map<String, Set<Dependency>> parseShadeOutput(Stream<String> lines) {
|
||||
return ParserUtils.parsePluginOutput(
|
||||
lines.filter(line -> !line.contains(" Excluding ")),
|
||||
SHADE_NEXT_MODULE_PATTERN,
|
||||
ShadeParser::parseBlock);
|
||||
}
|
||||
|
||||
private static Set<Dependency> parseBlock(Iterator<String> block) {
|
||||
final Set<Dependency> dependencies = new LinkedHashSet<>();
|
||||
|
||||
Optional<Dependency> parsedDependency = parseDependency(block.next());
|
||||
while (parsedDependency.isPresent()) {
|
||||
dependencies.add(parsedDependency.get());
|
||||
|
||||
if (block.hasNext()) {
|
||||
parsedDependency = parseDependency(block.next());
|
||||
} else {
|
||||
parsedDependency = Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
return dependencies;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static Optional<Dependency> parseDependency(String line) {
|
||||
Matcher dependencyMatcher = SHADE_INCLUDE_MODULE_PATTERN.matcher(line);
|
||||
if (!dependencyMatcher.find()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(
|
||||
Dependency.create(
|
||||
dependencyMatcher.group("groupId"),
|
||||
dependencyMatcher.group("artifactId"),
|
||||
dependencyMatcher.group("version"),
|
||||
dependencyMatcher.group("classifier")));
|
||||
}
|
||||
|
||||
private ShadeParser() {}
|
||||
}
|
||||
@ -0,0 +1,134 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shared;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Represents a dependency.
|
||||
*
|
||||
* <p>For some properties we return an {@link Optional}, for those that, depending on the plugin
|
||||
* goal, may not be determinable. For example, {@code dependency:copy} never prints the scope or
|
||||
* optional flag.
|
||||
*/
|
||||
public final class Dependency {
|
||||
|
||||
private final String groupId;
|
||||
private final String artifactId;
|
||||
private final String version;
|
||||
@Nullable private final String classifier;
|
||||
@Nullable private final String scope;
|
||||
@Nullable private final Boolean isOptional;
|
||||
|
||||
private Dependency(
|
||||
String groupId,
|
||||
String artifactId,
|
||||
String version,
|
||||
@Nullable String classifier,
|
||||
@Nullable String scope,
|
||||
@Nullable Boolean isOptional) {
|
||||
this.groupId = Objects.requireNonNull(groupId);
|
||||
this.artifactId = Objects.requireNonNull(artifactId);
|
||||
this.version = Objects.requireNonNull(version);
|
||||
this.classifier = classifier;
|
||||
this.scope = scope;
|
||||
this.isOptional = isOptional;
|
||||
}
|
||||
|
||||
public static Dependency create(
|
||||
String groupId,
|
||||
String artifactId,
|
||||
String version,
|
||||
String classifier,
|
||||
String scope,
|
||||
boolean isOptional) {
|
||||
return new Dependency(
|
||||
groupId,
|
||||
artifactId,
|
||||
version,
|
||||
classifier,
|
||||
Objects.requireNonNull(scope),
|
||||
isOptional);
|
||||
}
|
||||
|
||||
public static Dependency create(
|
||||
String groupId, String artifactId, String version, String classifier) {
|
||||
return new Dependency(groupId, artifactId, version, classifier, null, null);
|
||||
}
|
||||
|
||||
public String getGroupId() {
|
||||
return groupId;
|
||||
}
|
||||
|
||||
public String getArtifactId() {
|
||||
return artifactId;
|
||||
}
|
||||
|
||||
public String getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public Optional<String> getClassifier() {
|
||||
return Optional.ofNullable(classifier);
|
||||
}
|
||||
|
||||
public Optional<String> getScope() {
|
||||
return Optional.ofNullable(scope);
|
||||
}
|
||||
|
||||
public Optional<Boolean> isOptional() {
|
||||
return Optional.ofNullable(isOptional);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return groupId
|
||||
+ ":"
|
||||
+ artifactId
|
||||
+ ":"
|
||||
+ version
|
||||
+ (classifier != null ? ":" + classifier : "")
|
||||
+ (scope != null ? ":" + scope : "")
|
||||
+ (isOptional != null && isOptional ? " (optional)" : "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
Dependency that = (Dependency) o;
|
||||
return Objects.equals(groupId, that.groupId)
|
||||
&& Objects.equals(artifactId, that.artifactId)
|
||||
&& Objects.equals(version, that.version)
|
||||
&& Objects.equals(classifier, that.classifier)
|
||||
&& Objects.equals(scope, that.scope)
|
||||
&& Objects.equals(isOptional, that.isOptional);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(groupId, artifactId, version, classifier, scope, isOptional);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,137 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shared;
|
||||
|
||||
import org.apache.flink.annotation.VisibleForTesting;
|
||||
|
||||
import org.apache.flink.shaded.guava31.com.google.common.graph.Traverser;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
/**
|
||||
* Represents a dependency tree.
|
||||
*
|
||||
* <p>Every dependency can only occur exactly once.
|
||||
*/
|
||||
public class DependencyTree {
|
||||
|
||||
private final Map<String, Node> lookup = new LinkedHashMap<>();
|
||||
private final List<Node> directDependencies = new ArrayList<>();
|
||||
|
||||
public DependencyTree addDirectDependency(Dependency dependency) {
|
||||
final String key = getKey(dependency);
|
||||
if (lookup.containsKey(key)) {
|
||||
return this;
|
||||
}
|
||||
final Node node = new Node(dependency, null);
|
||||
|
||||
lookup.put(key, node);
|
||||
directDependencies.add(node);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public DependencyTree addTransitiveDependencyTo(
|
||||
Dependency transitiveDependency, Dependency parent) {
|
||||
final String key = getKey(transitiveDependency);
|
||||
if (lookup.containsKey(key)) {
|
||||
return this;
|
||||
}
|
||||
final Node node = lookup.get(getKey(parent)).addTransitiveDependency(transitiveDependency);
|
||||
|
||||
lookup.put(key, node);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
private static final class Node {
|
||||
private final Dependency dependency;
|
||||
@Nullable private final Node parent;
|
||||
private final List<Node> children = new ArrayList<>();
|
||||
|
||||
private Node(Dependency dependency, @Nullable Node parent) {
|
||||
this.dependency = dependency;
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
public Node addTransitiveDependency(Dependency dependency) {
|
||||
final Node node = new Node(dependency, this);
|
||||
this.children.add(node);
|
||||
return node;
|
||||
}
|
||||
|
||||
private boolean isRoot() {
|
||||
return parent == null;
|
||||
}
|
||||
}
|
||||
|
||||
public List<Dependency> getDirectDependencies() {
|
||||
return directDependencies.stream()
|
||||
.map(node -> node.dependency)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<Dependency> getPathTo(Dependency dependency) {
|
||||
final LinkedList<Dependency> path = new LinkedList<>();
|
||||
|
||||
Node node = lookup.get(getKey(dependency));
|
||||
path.addFirst(node.dependency);
|
||||
while (!node.isRoot()) {
|
||||
node = node.parent;
|
||||
path.addFirst(node.dependency);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
public Stream<Dependency> flatten() {
|
||||
return StreamSupport.stream(
|
||||
Traverser.<Node>forTree(node -> node.children)
|
||||
.depthFirstPreOrder(directDependencies)
|
||||
.spliterator(),
|
||||
false)
|
||||
.map(node -> node.dependency);
|
||||
}
|
||||
|
||||
/**
|
||||
* We don't use the {@link Dependency} as a key because we don't want lookups to be dependent on
|
||||
* scope or the optional flag.
|
||||
*
|
||||
* @param dependency
|
||||
* @return
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String getKey(Dependency dependency) {
|
||||
return dependency.getGroupId()
|
||||
+ ":"
|
||||
+ dependency.getArtifactId()
|
||||
+ ":"
|
||||
+ dependency.getVersion()
|
||||
+ ":"
|
||||
+ dependency.getClassifier().orElse("(no-classifier)");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shared;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/** Parsing utils. */
|
||||
public class ParserUtils {
|
||||
|
||||
/**
|
||||
* Iterates over the given lines, identifying plugin execution blocks with the given pattern and
|
||||
* parses the plugin output with the given parser.
|
||||
*
|
||||
* <p>This method assumes that the given pattern matches at most once for each module.
|
||||
*
|
||||
* <p>The given pattern must include a {@code module} group that captures the module that the
|
||||
* plugin runs on (without the scala suffix!).
|
||||
*
|
||||
* @param lines maven output lines
|
||||
* @param executionLinePattern pattern that matches plugin executions
|
||||
* @param blockParser parser for the plugin block
|
||||
* @return map containing the parser result for each module
|
||||
* @param <D> block parser output
|
||||
*/
|
||||
public static <D> Map<String, D> parsePluginOutput(
|
||||
Stream<String> lines,
|
||||
Pattern executionLinePattern,
|
||||
Function<Iterator<String>, D> blockParser) {
|
||||
final Map<String, D> result = new LinkedHashMap<>();
|
||||
|
||||
final Iterator<String> iterator = lines.iterator();
|
||||
|
||||
while (iterator.hasNext()) {
|
||||
Matcher moduleMatcher = executionLinePattern.matcher(iterator.next());
|
||||
while (!moduleMatcher.find()) {
|
||||
if (iterator.hasNext()) {
|
||||
moduleMatcher = executionLinePattern.matcher(iterator.next());
|
||||
} else {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
final String currentModule = moduleMatcher.group("module");
|
||||
|
||||
if (!iterator.hasNext()) {
|
||||
throw new IllegalStateException("Expected more output from the plugin.");
|
||||
}
|
||||
|
||||
result.put(currentModule, blockParser.apply(iterator));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
rootLogger.level = DEBUG
|
||||
rootLogger.appenderRef.console.ref = ConsoleAppender
|
||||
|
||||
appender.console.name = ConsoleAppender
|
||||
appender.console.type = CONSOLE
|
||||
appender.console.layout.type = PatternLayout
|
||||
appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
|
||||
@ -0,0 +1,20 @@
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
# This file lists modules which define additional dependencies, not shown by the maven shade plugin output.
|
||||
# These are usually undeclared shaded (or otherwise included) dependencies from transitive dependencies.
|
||||
@ -0,0 +1,525 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.licensecheck;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.file.FileSystem;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Tests for the {@link JarFileChecker}.
|
||||
*
|
||||
* <p>For dev purposes, generate a deploy-directory with: mvn clean deploy
|
||||
* -DaltDeploymentRepository=snapshot-repo::default::file:/tmp/flink-deployment -DskipTests
|
||||
* -Drat.skip and add a test checking that directory.
|
||||
*/
|
||||
class JarFileCheckerTest {
|
||||
|
||||
private static final List<String> VALID_NOTICE_PATH = Arrays.asList("META-INF", "NOTICE");
|
||||
private static final List<String> VALID_LICENSE_PATH = Arrays.asList("META-INF", "LICENSE");
|
||||
|
||||
@Test
|
||||
void testValidJarAccepted(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH))))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnMissingNoticeFile(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnInvalidNoticeFile(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(INVALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnNoticeFileInRoot(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_NOTICE_CONTENTS,
|
||||
Arrays.asList("some_custom_notice")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnMissingLicenseFile(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnInvalidLicenseFile(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(
|
||||
INVALID_LICENSE_CONTENTS, VALID_LICENSE_PATH))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnLicenseFileInRoot(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS,
|
||||
Arrays.asList("some_custom_license")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRejectedOnLicenseFileInSomeDirectory(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS,
|
||||
Arrays.asList(
|
||||
"some",
|
||||
"directory",
|
||||
"some_custom_license")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Disabled(
|
||||
"Currently not checked, but we may want to enforce this in the future to reduce ambiguity.")
|
||||
void testRejectedOnAdditionalLicenseFileInMetaInf(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
VALID_LICENSE_CONTENTS,
|
||||
Arrays.asList("META-INF", "LICENSE.txt")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIgnoreLicenseDirectories(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.directoryEntry(
|
||||
Arrays.asList("some", "license", "directory")))))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIgnoreClassFiles(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
"content",
|
||||
Arrays.asList("SomeLicenseClass.class")))))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIgnoreFtlFiles(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
"content", Arrays.asList("SomeLicenseFile.ftl")))))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIgnoreWebThirdPartyLicenses(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
"class contents",
|
||||
Arrays.asList("web", "3rdpartylicenses.txt")))))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testForbiddenLGPLongTextDetected(@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
"some GNU Lesser General public License text",
|
||||
Collections.singletonList("some_file.txt")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testForbiddenLGPMultiLineLongTextWithCommentAndLeadingWhitespaceDetected(
|
||||
@TempDir Path tempDir) throws Exception {
|
||||
assertThat(
|
||||
JarFileChecker.checkJar(
|
||||
createJar(
|
||||
tempDir,
|
||||
Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH),
|
||||
Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH),
|
||||
Entry.fileEntry(
|
||||
"some GNU Lesser General public \n\t\t//#License text",
|
||||
Collections.singletonList("some_file.txt")))))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
private static class Entry {
|
||||
final String contents;
|
||||
final List<String> path;
|
||||
final boolean isDirectory;
|
||||
|
||||
public static Entry directoryEntry(List<String> path) {
|
||||
return new Entry("", path, true);
|
||||
}
|
||||
|
||||
public static Entry fileEntry(String contents, List<String> path) {
|
||||
return new Entry(contents, path, false);
|
||||
}
|
||||
|
||||
private Entry(String contents, List<String> path, boolean isDirectory) {
|
||||
this.contents = contents;
|
||||
this.path = path;
|
||||
this.isDirectory = isDirectory;
|
||||
}
|
||||
}
|
||||
|
||||
private static Path createJar(Path tempDir, Entry... entries) throws Exception {
|
||||
final Path path = tempDir.resolve(UUID.randomUUID().toString() + ".jar");
|
||||
|
||||
final URI uriWithoutScheme = path.toUri();
|
||||
|
||||
final URI uri =
|
||||
new URI(
|
||||
"jar:file",
|
||||
uriWithoutScheme.getHost(),
|
||||
uriWithoutScheme.getPath(),
|
||||
uriWithoutScheme.getFragment());
|
||||
|
||||
// causes FileSystems#newFileSystem to automatically create a valid zip file
|
||||
// this is easier than manually creating a valid empty zip file manually
|
||||
final Map<String, String> env = new HashMap<>();
|
||||
env.put("create", "true");
|
||||
|
||||
try (FileSystem zip = FileSystems.newFileSystem(uri, env)) {
|
||||
// shortcut to getting the single root
|
||||
final Path root = zip.getPath("").toAbsolutePath();
|
||||
for (Entry entry : entries) {
|
||||
final Path zipPath =
|
||||
root.resolve(
|
||||
zip.getPath(
|
||||
entry.path.get(0),
|
||||
entry.path
|
||||
.subList(1, entry.path.size())
|
||||
.toArray(new String[] {})));
|
||||
if (entry.isDirectory) {
|
||||
Files.createDirectories(zipPath);
|
||||
} else {
|
||||
Files.createDirectories(zipPath.getParent());
|
||||
Files.write(zipPath, entry.contents.getBytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
private static final String INVALID_NOTICE_CONTENTS = "" + "invalid";
|
||||
|
||||
private static final String VALID_NOTICE_CONTENTS =
|
||||
""
|
||||
+ "Flink : SomeModule\n"
|
||||
+ "Copyright 2014-2020 The Apache Software Foundation\n"
|
||||
+ "\n"
|
||||
+ "This product includes software developed at\n"
|
||||
+ "The Apache Software Foundation (http://www.apache.org/).";
|
||||
|
||||
private static final String INVALID_LICENSE_CONTENTS = "" + "invalid";
|
||||
|
||||
private static final String VALID_LICENSE_CONTENTS =
|
||||
""
|
||||
+ "\n"
|
||||
+ " Apache License\n"
|
||||
+ " Version 2.0, January 2004\n"
|
||||
+ " http://www.apache.org/licenses/\n"
|
||||
+ "\n"
|
||||
+ " TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n"
|
||||
+ "\n"
|
||||
+ " 1. Definitions.\n"
|
||||
+ "\n"
|
||||
+ " \"License\" shall mean the terms and conditions for use, reproduction,\n"
|
||||
+ " and distribution as defined by Sections 1 through 9 of this document.\n"
|
||||
+ "\n"
|
||||
+ " \"Licensor\" shall mean the copyright owner or entity authorized by\n"
|
||||
+ " the copyright owner that is granting the License.\n"
|
||||
+ "\n"
|
||||
+ " \"Legal Entity\" shall mean the union of the acting entity and all\n"
|
||||
+ " other entities that control, are controlled by, or are under common\n"
|
||||
+ " control with that entity. For the purposes of this definition,\n"
|
||||
+ " \"control\" means (i) the power, direct or indirect, to cause the\n"
|
||||
+ " direction or management of such entity, whether by contract or\n"
|
||||
+ " otherwise, or (ii) ownership of fifty percent (50%) or more of the\n"
|
||||
+ " outstanding shares, or (iii) beneficial ownership of such entity.\n"
|
||||
+ "\n"
|
||||
+ " \"You\" (or \"Your\") shall mean an individual or Legal Entity\n"
|
||||
+ " exercising permissions granted by this License.\n"
|
||||
+ "\n"
|
||||
+ " \"Source\" form shall mean the preferred form for making modifications,\n"
|
||||
+ " including but not limited to software source code, documentation\n"
|
||||
+ " source, and configuration files.\n"
|
||||
+ "\n"
|
||||
+ " \"Object\" form shall mean any form resulting from mechanical\n"
|
||||
+ " transformation or translation of a Source form, including but\n"
|
||||
+ " not limited to compiled object code, generated documentation,\n"
|
||||
+ " and conversions to other media types.\n"
|
||||
+ "\n"
|
||||
+ " \"Work\" shall mean the work of authorship, whether in Source or\n"
|
||||
+ " Object form, made available under the License, as indicated by a\n"
|
||||
+ " copyright notice that is included in or attached to the work\n"
|
||||
+ " (an example is provided in the Appendix below).\n"
|
||||
+ "\n"
|
||||
+ " \"Derivative Works\" shall mean any work, whether in Source or Object\n"
|
||||
+ " form, that is based on (or derived from) the Work and for which the\n"
|
||||
+ " editorial revisions, annotations, elaborations, or other modifications\n"
|
||||
+ " represent, as a whole, an original work of authorship. For the purposes\n"
|
||||
+ " of this License, Derivative Works shall not include works that remain\n"
|
||||
+ " separable from, or merely link (or bind by name) to the interfaces of,\n"
|
||||
+ " the Work and Derivative Works thereof.\n"
|
||||
+ "\n"
|
||||
+ " \"Contribution\" shall mean any work of authorship, including\n"
|
||||
+ " the original version of the Work and any modifications or additions\n"
|
||||
+ " to that Work or Derivative Works thereof, that is intentionally\n"
|
||||
+ " submitted to Licensor for inclusion in the Work by the copyright owner\n"
|
||||
+ " or by an individual or Legal Entity authorized to submit on behalf of\n"
|
||||
+ " the copyright owner. For the purposes of this definition, \"submitted\"\n"
|
||||
+ " means any form of electronic, verbal, or written communication sent\n"
|
||||
+ " to the Licensor or its representatives, including but not limited to\n"
|
||||
+ " communication on electronic mailing lists, source code control systems,\n"
|
||||
+ " and issue tracking systems that are managed by, or on behalf of, the\n"
|
||||
+ " Licensor for the purpose of discussing and improving the Work, but\n"
|
||||
+ " excluding communication that is conspicuously marked or otherwise\n"
|
||||
+ " designated in writing by the copyright owner as \"Not a Contribution.\"\n"
|
||||
+ "\n"
|
||||
+ " \"Contributor\" shall mean Licensor and any individual or Legal Entity\n"
|
||||
+ " on behalf of whom a Contribution has been received by Licensor and\n"
|
||||
+ " subsequently incorporated within the Work.\n"
|
||||
+ "\n"
|
||||
+ " 2. Grant of Copyright License. Subject to the terms and conditions of\n"
|
||||
+ " this License, each Contributor hereby grants to You a perpetual,\n"
|
||||
+ " worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n"
|
||||
+ " copyright license to reproduce, prepare Derivative Works of,\n"
|
||||
+ " publicly display, publicly perform, sublicense, and distribute the\n"
|
||||
+ " Work and such Derivative Works in Source or Object form.\n"
|
||||
+ "\n"
|
||||
+ " 3. Grant of Patent License. Subject to the terms and conditions of\n"
|
||||
+ " this License, each Contributor hereby grants to You a perpetual,\n"
|
||||
+ " worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n"
|
||||
+ " (except as stated in this section) patent license to make, have made,\n"
|
||||
+ " use, offer to sell, sell, import, and otherwise transfer the Work,\n"
|
||||
+ " where such license applies only to those patent claims licensable\n"
|
||||
+ " by such Contributor that are necessarily infringed by their\n"
|
||||
+ " Contribution(s) alone or by combination of their Contribution(s)\n"
|
||||
+ " with the Work to which such Contribution(s) was submitted. If You\n"
|
||||
+ " institute patent litigation against any entity (including a\n"
|
||||
+ " cross-claim or counterclaim in a lawsuit) alleging that the Work\n"
|
||||
+ " or a Contribution incorporated within the Work constitutes direct\n"
|
||||
+ " or contributory patent infringement, then any patent licenses\n"
|
||||
+ " granted to You under this License for that Work shall terminate\n"
|
||||
+ " as of the date such litigation is filed.\n"
|
||||
+ "\n"
|
||||
+ " 4. Redistribution. You may reproduce and distribute copies of the\n"
|
||||
+ " Work or Derivative Works thereof in any medium, with or without\n"
|
||||
+ " modifications, and in Source or Object form, provided that You\n"
|
||||
+ " meet the following conditions:\n"
|
||||
+ "\n"
|
||||
+ " (a) You must give any other recipients of the Work or\n"
|
||||
+ " Derivative Works a copy of this License; and\n"
|
||||
+ "\n"
|
||||
+ " (b) You must cause any modified files to carry prominent notices\n"
|
||||
+ " stating that You changed the files; and\n"
|
||||
+ "\n"
|
||||
+ " (c) You must retain, in the Source form of any Derivative Works\n"
|
||||
+ " that You distribute, all copyright, patent, trademark, and\n"
|
||||
+ " attribution notices from the Source form of the Work,\n"
|
||||
+ " excluding those notices that do not pertain to any part of\n"
|
||||
+ " the Derivative Works; and\n"
|
||||
+ "\n"
|
||||
+ " (d) If the Work includes a \"NOTICE\" text file as part of its\n"
|
||||
+ " distribution, then any Derivative Works that You distribute must\n"
|
||||
+ " include a readable copy of the attribution notices contained\n"
|
||||
+ " within such NOTICE file, excluding those notices that do not\n"
|
||||
+ " pertain to any part of the Derivative Works, in at least one\n"
|
||||
+ " of the following places: within a NOTICE text file distributed\n"
|
||||
+ " as part of the Derivative Works; within the Source form or\n"
|
||||
+ " documentation, if provided along with the Derivative Works; or,\n"
|
||||
+ " within a display generated by the Derivative Works, if and\n"
|
||||
+ " wherever such third-party notices normally appear. The contents\n"
|
||||
+ " of the NOTICE file are for informational purposes only and\n"
|
||||
+ " do not modify the License. You may add Your own attribution\n"
|
||||
+ " notices within Derivative Works that You distribute, alongside\n"
|
||||
+ " or as an addendum to the NOTICE text from the Work, provided\n"
|
||||
+ " that such additional attribution notices cannot be construed\n"
|
||||
+ " as modifying the License.\n"
|
||||
+ "\n"
|
||||
+ " You may add Your own copyright statement to Your modifications and\n"
|
||||
+ " may provide additional or different license terms and conditions\n"
|
||||
+ " for use, reproduction, or distribution of Your modifications, or\n"
|
||||
+ " for any such Derivative Works as a whole, provided Your use,\n"
|
||||
+ " reproduction, and distribution of the Work otherwise complies with\n"
|
||||
+ " the conditions stated in this License.\n"
|
||||
+ "\n"
|
||||
+ " 5. Submission of Contributions. Unless You explicitly state otherwise,\n"
|
||||
+ " any Contribution intentionally submitted for inclusion in the Work\n"
|
||||
+ " by You to the Licensor shall be under the terms and conditions of\n"
|
||||
+ " this License, without any additional terms or conditions.\n"
|
||||
+ " Notwithstanding the above, nothing herein shall supersede or modify\n"
|
||||
+ " the terms of any separate license agreement you may have executed\n"
|
||||
+ " with Licensor regarding such Contributions.\n"
|
||||
+ "\n"
|
||||
+ " 6. Trademarks. This License does not grant permission to use the trade\n"
|
||||
+ " names, trademarks, service marks, or product names of the Licensor,\n"
|
||||
+ " except as required for reasonable and customary use in describing the\n"
|
||||
+ " origin of the Work and reproducing the content of the NOTICE file.\n"
|
||||
+ "\n"
|
||||
+ " 7. Disclaimer of Warranty. Unless required by applicable law or\n"
|
||||
+ " agreed to in writing, Licensor provides the Work (and each\n"
|
||||
+ " Contributor provides its Contributions) on an \"AS IS\" BASIS,\n"
|
||||
+ " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n"
|
||||
+ " implied, including, without limitation, any warranties or conditions\n"
|
||||
+ " of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n"
|
||||
+ " PARTICULAR PURPOSE. You are solely responsible for determining the\n"
|
||||
+ " appropriateness of using or redistributing the Work and assume any\n"
|
||||
+ " risks associated with Your exercise of permissions under this License.\n"
|
||||
+ "\n"
|
||||
+ " 8. Limitation of Liability. In no event and under no legal theory,\n"
|
||||
+ " whether in tort (including negligence), contract, or otherwise,\n"
|
||||
+ " unless required by applicable law (such as deliberate and grossly\n"
|
||||
+ " negligent acts) or agreed to in writing, shall any Contributor be\n"
|
||||
+ " liable to You for damages, including any direct, indirect, special,\n"
|
||||
+ " incidental, or consequential damages of any character arising as a\n"
|
||||
+ " result of this License or out of the use or inability to use the\n"
|
||||
+ " Work (including but not limited to damages for loss of goodwill,\n"
|
||||
+ " work stoppage, computer failure or malfunction, or any and all\n"
|
||||
+ " other commercial damages or losses), even if such Contributor\n"
|
||||
+ " has been advised of the possibility of such damages.\n"
|
||||
+ "\n"
|
||||
+ " 9. Accepting Warranty or Additional Liability. While redistributing\n"
|
||||
+ " the Work or Derivative Works thereof, You may choose to offer,\n"
|
||||
+ " and charge a fee for, acceptance of support, warranty, indemnity,\n"
|
||||
+ " or other liability obligations and/or rights consistent with this\n"
|
||||
+ " License. However, in accepting such obligations, You may act only\n"
|
||||
+ " on Your own behalf and on Your sole responsibility, not on behalf\n"
|
||||
+ " of any other Contributor, and only if You agree to indemnify,\n"
|
||||
+ " defend, and hold each Contributor harmless for any liability\n"
|
||||
+ " incurred by, or claims asserted against, such Contributor by reason\n"
|
||||
+ " of your accepting any such warranty or additional liability.\n"
|
||||
+ "\n"
|
||||
+ " END OF TERMS AND CONDITIONS\n"
|
||||
+ "\n"
|
||||
+ " APPENDIX: How to apply the Apache License to your work.\n"
|
||||
+ "\n"
|
||||
+ " To apply the Apache License to your work, attach the following\n"
|
||||
+ " boilerplate notice, with the fields enclosed by brackets \"[]\"\n"
|
||||
+ " replaced with your own identifying information. (Don't include\n"
|
||||
+ " the brackets!) The text should be enclosed in the appropriate\n"
|
||||
+ " comment syntax for the file format. We also recommend that a\n"
|
||||
+ " file or class name and description of purpose be included on the\n"
|
||||
+ " same \"printed page\" as the copyright notice for easier\n"
|
||||
+ " identification within third-party archives.\n"
|
||||
+ "\n"
|
||||
+ " Copyright [yyyy] [name of copyright owner]\n"
|
||||
+ "\n"
|
||||
+ " Licensed under the Apache License, Version 2.0 (the \"License\");\n"
|
||||
+ " you may not use this file except in compliance with the License.\n"
|
||||
+ " You may obtain a copy of the License at\n"
|
||||
+ "\n"
|
||||
+ " http://www.apache.org/licenses/LICENSE-2.0\n"
|
||||
+ "\n"
|
||||
+ " Unless required by applicable law or agreed to in writing, software\n"
|
||||
+ " distributed under the License is distributed on an \"AS IS\" BASIS,\n"
|
||||
+ " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"
|
||||
+ " See the License for the specific language governing permissions and\n"
|
||||
+ " limitations under the License.\n";
|
||||
}
|
||||
@ -0,0 +1,209 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.licensecheck;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.notice.NoticeContents;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class NoticeFileCheckerTest {
|
||||
@Test
|
||||
void testRunHappyPath() throws IOException {
|
||||
final String moduleName = "test";
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(moduleName, Collections.singleton(bundledDependency));
|
||||
final Set<String> deployedModules = Collections.singleton(moduleName);
|
||||
final Optional<NoticeContents> noticeContents =
|
||||
Optional.of(
|
||||
new NoticeContents(
|
||||
moduleName, Collections.singletonList(bundledDependency)));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.run(
|
||||
bundleDependencies,
|
||||
deployedModules,
|
||||
Collections.singletonMap(moduleName, noticeContents)))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunRejectsMissingNotice() throws IOException {
|
||||
final String moduleName = "test";
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(moduleName, Collections.singleton(bundledDependency));
|
||||
final Set<String> deployedModules = Collections.singleton(moduleName);
|
||||
final Optional<NoticeContents> missingNotice = Optional.empty();
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.run(
|
||||
bundleDependencies,
|
||||
deployedModules,
|
||||
Collections.singletonMap(moduleName, missingNotice)))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunRejectsIncorrectNotice() throws IOException {
|
||||
final String moduleName = "test";
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(moduleName, Collections.singleton(bundledDependency));
|
||||
final Set<String> deployedModules = Collections.singleton(moduleName);
|
||||
final Optional<NoticeContents> emptyNotice =
|
||||
Optional.of(new NoticeContents(moduleName, Collections.emptyList()));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.run(
|
||||
bundleDependencies,
|
||||
deployedModules,
|
||||
Collections.singletonMap(moduleName, emptyNotice)))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunSkipsNonDeployedModules() throws IOException {
|
||||
final String moduleName = "test";
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(moduleName, Collections.singleton(bundledDependency));
|
||||
final Set<String> deployedModules = Collections.emptySet();
|
||||
// this would usually be a problem, but since the module is not deployed it's OK!
|
||||
final Optional<NoticeContents> emptyNotice =
|
||||
Optional.of(new NoticeContents(moduleName, Collections.emptyList()));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.run(
|
||||
bundleDependencies,
|
||||
deployedModules,
|
||||
Collections.singletonMap(moduleName, emptyNotice)))
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunIncludesBundledNonDeployedModules() throws IOException {
|
||||
final Map<String, Set<Dependency>> bundledDependencies = new HashMap<>();
|
||||
final Map<String, Optional<NoticeContents>> notices = new HashMap<>();
|
||||
|
||||
// a module that is not deployed but bundles another dependency with an empty NOTICE
|
||||
final String nonDeployedModuleName = "nonDeployed";
|
||||
final Dependency nonDeployedDependency =
|
||||
Dependency.create("a", nonDeployedModuleName, "c", null);
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
bundledDependencies.put(nonDeployedModuleName, Collections.singleton(bundledDependency));
|
||||
// this would usually not be a problem, but since the module is not bundled it's not OK!
|
||||
final Optional<NoticeContents> emptyNotice =
|
||||
Optional.of(new NoticeContents(nonDeployedModuleName, Collections.emptyList()));
|
||||
notices.put(nonDeployedModuleName, emptyNotice);
|
||||
|
||||
// a module that is deploys and bundles the above
|
||||
final String bundlingModule = "bundling";
|
||||
bundledDependencies.put(bundlingModule, Collections.singleton(nonDeployedDependency));
|
||||
final Optional<NoticeContents> correctNotice =
|
||||
Optional.of(
|
||||
new NoticeContents(
|
||||
bundlingModule, Collections.singletonList(nonDeployedDependency)));
|
||||
notices.put(bundlingModule, correctNotice);
|
||||
|
||||
final Set<String> deployedModules = Collections.singleton(bundlingModule);
|
||||
|
||||
assertThat(NoticeFileChecker.run(bundledDependencies, deployedModules, notices))
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCheckNoticeFileHappyPath() {
|
||||
final String moduleName = "test";
|
||||
final Dependency bundledDependency = Dependency.create("a", "b", "c", null);
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(moduleName, Collections.singleton(bundledDependency));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.checkNoticeFile(
|
||||
bundleDependencies,
|
||||
moduleName,
|
||||
new NoticeContents(
|
||||
moduleName, Collections.singletonList(bundledDependency))))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCheckNoticeFileRejectsEmptyFile() {
|
||||
assertThat(NoticeFileChecker.checkNoticeFile(Collections.emptyMap(), "test", null))
|
||||
.containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCheckNoticeFileToleratesModuleNameMismatch() {
|
||||
final String moduleName = "test";
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.checkNoticeFile(
|
||||
Collections.emptyMap(),
|
||||
moduleName,
|
||||
new NoticeContents(moduleName + "2", Collections.emptyList())))
|
||||
.containsOnlyKeys(NoticeFileChecker.Severity.TOLERATED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCheckNoticeFileRejectsDuplicateLine() {
|
||||
final String moduleName = "test";
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(
|
||||
moduleName, Collections.singleton(Dependency.create("a", "b", "c", null)));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.checkNoticeFile(
|
||||
bundleDependencies,
|
||||
moduleName,
|
||||
new NoticeContents(
|
||||
moduleName,
|
||||
Arrays.asList(
|
||||
Dependency.create("a", "b", "c", null),
|
||||
Dependency.create("a", "b", "c", null)))))
|
||||
.containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCheckNoticeFileRejectsMissingDependency() {
|
||||
final String moduleName = "test";
|
||||
final Map<String, Set<Dependency>> bundleDependencies = new HashMap<>();
|
||||
bundleDependencies.put(
|
||||
moduleName, Collections.singleton(Dependency.create("a", "b", "c", null)));
|
||||
|
||||
assertThat(
|
||||
NoticeFileChecker.checkNoticeFile(
|
||||
bundleDependencies,
|
||||
moduleName,
|
||||
new NoticeContents(moduleName, Collections.emptyList())))
|
||||
.containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.optional;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ShadeOptionalCheckerTest {
|
||||
private static final String MODULE = "module";
|
||||
|
||||
@Test
|
||||
void testNonBundledDependencyIsIgnored() {
|
||||
final Dependency dependency = createMandatoryDependency("a");
|
||||
final Set<Dependency> bundled = Collections.emptySet();
|
||||
final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNonBundledDependencyIsIgnoredEvenIfOthersAreBundled() {
|
||||
final Dependency dependencyA = createMandatoryDependency("a");
|
||||
final Dependency dependencyB = createMandatoryDependency("B");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addDirectDependency(dependencyB);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).containsExactly(dependencyB);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDirectBundledOptionalDependencyIsAccepted() {
|
||||
final Dependency dependency = createOptionalDependency("a");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependency);
|
||||
final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDirectBundledDependencyMustBeOptional() {
|
||||
final Dependency dependency = createMandatoryDependency("a");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependency);
|
||||
final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).containsExactly(dependency);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransitiveBundledOptionalDependencyIsAccepted() {
|
||||
final Dependency dependencyA = createMandatoryDependency("a");
|
||||
final Dependency dependencyB = createOptionalDependency("b");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addTransitiveDependencyTo(dependencyB, dependencyA);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransitiveBundledDependencyMustBeOptional() {
|
||||
final Dependency dependencyA = createMandatoryDependency("a");
|
||||
final Dependency dependencyB = createMandatoryDependency("b");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addTransitiveDependencyTo(dependencyB, dependencyA);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).containsExactly(dependencyB);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransitiveBundledDependencyMayNotBeOptionalIfParentIsOptional() {
|
||||
final Dependency dependencyA = createOptionalDependency("a");
|
||||
final Dependency dependencyB = createMandatoryDependency("b");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addTransitiveDependencyTo(dependencyB, dependencyA);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransitiveBundledDependencyMayNotBeOptionalIfParentHasTestScope() {
|
||||
final Dependency dependencyA = createTestDependency("a");
|
||||
final Dependency dependencyB = createMandatoryDependency("b");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addTransitiveDependencyTo(dependencyB, dependencyA);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTransitiveBundledDependencyMayNotBeOptionalIfParentHasProvidedScope() {
|
||||
final Dependency dependencyA = createProvidedDependency("a");
|
||||
final Dependency dependencyB = createMandatoryDependency("b");
|
||||
final Set<Dependency> bundled = Collections.singleton(dependencyB);
|
||||
final DependencyTree dependencyTree =
|
||||
new DependencyTree()
|
||||
.addDirectDependency(dependencyA)
|
||||
.addTransitiveDependencyTo(dependencyB, dependencyA);
|
||||
|
||||
final Set<Dependency> violations =
|
||||
ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree);
|
||||
|
||||
assertThat(violations).isEmpty();
|
||||
}
|
||||
|
||||
private static Dependency createMandatoryDependency(String artifactId) {
|
||||
return Dependency.create("groupId", artifactId, "version", null);
|
||||
}
|
||||
|
||||
private static Dependency createOptionalDependency(String artifactId) {
|
||||
return Dependency.create("groupId", artifactId, "version", null, "compile", true);
|
||||
}
|
||||
|
||||
private static Dependency createProvidedDependency(String artifactId) {
|
||||
return Dependency.create("groupId", artifactId, "version", null, "provided", false);
|
||||
}
|
||||
|
||||
private static Dependency createTestDependency(String artifactId) {
|
||||
return Dependency.create("groupId", artifactId, "version", null, "test", false);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.dependency;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** Tests the parsing of {@code dependency:copy}. */
|
||||
class DependencyParserCopyTest {
|
||||
|
||||
private static Stream<String> getTestDependencyCopy() {
|
||||
return Stream.of(
|
||||
"[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m1 ---",
|
||||
"[INFO] Configured Artifact: external:dependency1:2.1:jar",
|
||||
"[INFO] Configured Artifact: external:dependency4:classifier:2.4:jar",
|
||||
"[INFO] Copying dependency1-2.1.jar to /some/path/dependency1-2.1.jar",
|
||||
"[INFO] Copying dependency4-2.4.jar to /some/path/dependency4-2.4.jar",
|
||||
"[INFO]",
|
||||
"[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m2 ---",
|
||||
"[INFO] Configured Artifact: internal:m1:1.1:jar",
|
||||
"[INFO] Copying internal-1.1.jar to /some/path/m1-1.1.jar");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyParsing() {
|
||||
final Map<String, Set<Dependency>> dependenciesByModule =
|
||||
DependencyParser.parseDependencyCopyOutput(getTestDependencyCopy());
|
||||
|
||||
assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2");
|
||||
assertThat(dependenciesByModule.get("m1"))
|
||||
.containsExactlyInAnyOrder(
|
||||
Dependency.create("external", "dependency1", "2.1", null),
|
||||
Dependency.create("external", "dependency4", "2.4", "classifier"));
|
||||
assertThat(dependenciesByModule.get("m2"))
|
||||
.containsExactlyInAnyOrder(Dependency.create("internal", "m1", "1.1", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingGroupId() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:jar"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getGroupId()).isEqualTo("external"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingArtifactId() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:jar"))
|
||||
.hasValueSatisfying(
|
||||
dependency ->
|
||||
assertThat(dependency.getArtifactId()).isEqualTo("dependency1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingVersion() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:jar"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingScope() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:jar"))
|
||||
.hasValueSatisfying(dependency -> assertThat(dependency.getScope()).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingOptional() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:jar"))
|
||||
.hasValueSatisfying(dependency -> assertThat(dependency.isOptional()).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingWithNonJarType() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:1.0:pom"))
|
||||
.hasValue(Dependency.create("external", "dependency1", "1.0", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCopyLineParsingClassifier() {
|
||||
assertThat(
|
||||
DependencyParser.parseCopyDependency(
|
||||
"[INFO] Configured Artifact: external:dependency1:some_classifier:1.0:jar"))
|
||||
.hasValueSatisfying(
|
||||
dependency ->
|
||||
assertThat(dependency.getClassifier()).hasValue("some_classifier"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.dependency;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** Tests the parsing of {@code dependency:tree}. */
|
||||
class DependencyParserTreeTest {
|
||||
|
||||
private static Stream<String> getTestDependencyTree() {
|
||||
return Stream.of(
|
||||
"[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m1 ---",
|
||||
"[INFO] internal:m1:jar:1.1",
|
||||
"[INFO] +- external:dependency1:jar:2.1:compile",
|
||||
"[INFO] | +- external:dependency2:jar:2.2:compile (optional)",
|
||||
"[INFO] | | \\- external:dependency3:jar:2.3:provided",
|
||||
"[INFO] | +- external:dependency4:jar:classifier:2.4:compile",
|
||||
"[INFO]",
|
||||
"[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m2 ---",
|
||||
"[INFO] internal:m2:jar:1.2",
|
||||
"[INFO] +- internal:m1:jar:1.1:compile",
|
||||
"[INFO] | +- external:dependency4:jar:2.4:compile");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeParsing() {
|
||||
final Map<String, DependencyTree> dependenciesByModule =
|
||||
DependencyParser.parseDependencyTreeOutput(getTestDependencyTree());
|
||||
|
||||
assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2");
|
||||
assertThat(dependenciesByModule.get("m1").flatten())
|
||||
.containsExactlyInAnyOrder(
|
||||
Dependency.create("external", "dependency1", "2.1", null, "compile", false),
|
||||
Dependency.create("external", "dependency2", "2.2", null, "compile", true),
|
||||
Dependency.create(
|
||||
"external", "dependency3", "2.3", null, "provided", false),
|
||||
Dependency.create(
|
||||
"external", "dependency4", "2.4", "classifier", "compile", false));
|
||||
assertThat(dependenciesByModule.get("m2").flatten())
|
||||
.containsExactlyInAnyOrder(
|
||||
Dependency.create("internal", "m1", "1.1", null, "compile", false),
|
||||
Dependency.create(
|
||||
"external", "dependency4", "2.4", null, "compile", false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingGroupId() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:compile"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getGroupId()).isEqualTo("external"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingArtifactId() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:compile"))
|
||||
.hasValueSatisfying(
|
||||
dependency ->
|
||||
assertThat(dependency.getArtifactId()).isEqualTo("dependency1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingVersion() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:compile"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingScope() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:provided"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getScope()).hasValue("provided"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingWithNonJarType() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:pom:1.0:compile"))
|
||||
.hasValue(
|
||||
Dependency.create(
|
||||
"external", "dependency1", "1.0", null, "compile", false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingWithClassifier() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:some_classifier:1.0:compile"))
|
||||
.hasValue(
|
||||
Dependency.create(
|
||||
"external",
|
||||
"dependency1",
|
||||
"1.0",
|
||||
"some_classifier",
|
||||
"compile",
|
||||
false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingWithoutOptional() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:compile"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.isOptional()).hasValue(false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTreeLineParsingWithOptional() {
|
||||
assertThat(
|
||||
DependencyParser.parseTreeDependency(
|
||||
"[INFO] +- external:dependency1:jar:1.0:compile (optional)"))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.isOptional()).hasValue(true));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.deploy;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class DeployParserTest {
|
||||
@Test
|
||||
void testParseDeployOutputDetectsDeployment() {
|
||||
assertThat(
|
||||
DeployParser.parseDeployOutput(
|
||||
Stream.of(
|
||||
"[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---",
|
||||
"[INFO] ")))
|
||||
.containsExactly("flink-parent");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseDeployOutputDetectsDeploymentWithAltRepository() {
|
||||
assertThat(
|
||||
DeployParser.parseDeployOutput(
|
||||
Stream.of(
|
||||
"[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---",
|
||||
"[INFO] Using alternate deployment repository.../tmp/flink-validation-deployment")))
|
||||
.containsExactly("flink-parent");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseDeployOutputDetectsSkippedDeployments() {
|
||||
assertThat(
|
||||
DeployParser.parseDeployOutput(
|
||||
Stream.of(
|
||||
"[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---",
|
||||
"[INFO] Skipping artifact deployment")))
|
||||
.isEmpty();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.notice;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class NoticeParserTest {
|
||||
@Test
|
||||
void testParseNoticeFileCommonPath() {
|
||||
final String module = "some-module";
|
||||
final Dependency dependency1 =
|
||||
Dependency.create("groupId1", "artifactId1", "version1", null);
|
||||
final Dependency dependency2 =
|
||||
Dependency.create("groupId2", "artifactId2", "version2", "classifier2");
|
||||
final Dependency dependency3 =
|
||||
Dependency.create("org.codehaus.woodstox", "stax2-api", "4.2.1", null);
|
||||
final List<String> noticeContents =
|
||||
Arrays.asList(
|
||||
module,
|
||||
"",
|
||||
"Some text about the applicable license",
|
||||
"- groupId1:artifactId1:version1",
|
||||
"- groupId2:artifactId2:classifier2:version2",
|
||||
"- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)",
|
||||
"",
|
||||
"some epilogue");
|
||||
|
||||
assertThat(NoticeParser.parseNoticeFile(noticeContents))
|
||||
.hasValueSatisfying(
|
||||
contents -> {
|
||||
assertThat(contents.getNoticeModuleName()).isEqualTo(module);
|
||||
assertThat(contents.getDeclaredDependencies())
|
||||
.containsExactlyInAnyOrder(
|
||||
dependency1, dependency2, dependency3);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseNoticeFileBundlesPath() {
|
||||
final String module = "some-module";
|
||||
final Dependency dependency =
|
||||
Dependency.create("groupId", "artifactId", "version", "classifier");
|
||||
final List<String> noticeContents =
|
||||
Arrays.asList(
|
||||
module, "", "Something bundles \"groupId:artifactId:classifier:version\"");
|
||||
|
||||
assertThat(NoticeParser.parseNoticeFile(noticeContents))
|
||||
.hasValueSatisfying(
|
||||
contents -> {
|
||||
assertThat(contents.getNoticeModuleName()).isEqualTo(module);
|
||||
assertThat(contents.getDeclaredDependencies())
|
||||
.containsExactlyInAnyOrder(dependency);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseNoticeFileMalformedDependencyIgnored() {
|
||||
final String module = "some-module";
|
||||
final Dependency dependency = Dependency.create("groupId", "artifactId", "version", null);
|
||||
final List<String> noticeContents = Arrays.asList(module, "- " + dependency, "- a:b");
|
||||
|
||||
assertThat(NoticeParser.parseNoticeFile(noticeContents))
|
||||
.hasValueSatisfying(
|
||||
contents -> {
|
||||
assertThat(contents.getNoticeModuleName()).isEqualTo(module);
|
||||
assertThat(contents.getDeclaredDependencies())
|
||||
.containsExactlyInAnyOrder(dependency);
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shade;
|
||||
|
||||
import org.apache.flink.cdc.tools.ci.utils.shared.Dependency;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ShadeParserTest {
|
||||
|
||||
private static Stream<String> getTestDependencyCopy() {
|
||||
return Stream.of(
|
||||
"[INFO] --- maven-shade-plugin:3.2.0:shade (shade-flink) @ m1 ---",
|
||||
"[INFO] Including external:dependency1:jar:2.1 in the shaded jar.",
|
||||
"[INFO] Excluding external:dependency3:jar:2.3 from the shaded jar.",
|
||||
"[INFO] Including external:dependency4:jar:classifier:2.4 in the shaded jar.",
|
||||
"[INFO] Replacing original artifact with shaded artifact.",
|
||||
"[INFO] Replacing /some/path/m1.jar with /some/path/m1-shaded.jar",
|
||||
"[INFO]",
|
||||
"[INFO] --- maven-shade-plugin:3.2.0:shade (shade-flink) @ m2 ---",
|
||||
"[INFO] Including internal:m1:jar:1.1 in the shaded jar.",
|
||||
"[INFO] Replacing /some/path/m2.jar with /some/path/m2-shaded.jar");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParsing() {
|
||||
final Map<String, Set<Dependency>> dependenciesByModule =
|
||||
ShadeParser.parseShadeOutput(getTestDependencyCopy());
|
||||
|
||||
assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2");
|
||||
assertThat(dependenciesByModule.get("m1"))
|
||||
.containsExactlyInAnyOrder(
|
||||
Dependency.create("external", "dependency1", "2.1", null),
|
||||
Dependency.create("external", "dependency4", "2.4", "classifier"));
|
||||
assertThat(dependenciesByModule.get("m2"))
|
||||
.containsExactlyInAnyOrder(Dependency.create("internal", "m1", "1.1", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLineParsingGroupId() {
|
||||
assertThat(
|
||||
ShadeParser.parseDependency(
|
||||
"Including external:dependency1:jar:1.0 in the shaded jar."))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getGroupId()).isEqualTo("external"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLineParsingArtifactId() {
|
||||
assertThat(
|
||||
ShadeParser.parseDependency(
|
||||
"Including external:dependency1:jar:1.0 in the shaded jar."))
|
||||
.hasValueSatisfying(
|
||||
dependency ->
|
||||
assertThat(dependency.getArtifactId()).isEqualTo("dependency1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLineParsingVersion() {
|
||||
assertThat(
|
||||
ShadeParser.parseDependency(
|
||||
"Including external:dependency1:jar:1.0 in the shaded jar."))
|
||||
.hasValueSatisfying(
|
||||
dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLineParsingWithNonJarType() {
|
||||
assertThat(
|
||||
ShadeParser.parseDependency(
|
||||
"Including external:dependency1:pom:1.0 in the shaded jar."))
|
||||
.isPresent();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,96 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.flink.cdc.tools.ci.utils.shared;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class DependencyTreeTest {
|
||||
private static final Dependency DEPENDENCY =
|
||||
Dependency.create("groupId", "artifactId", "version", null);
|
||||
|
||||
@Test
|
||||
void testDependencyKeyIncludesGroupId() {
|
||||
testDependencyKeyInclusion(
|
||||
Dependency.create(
|
||||
"xxx",
|
||||
DEPENDENCY.getArtifactId(),
|
||||
DEPENDENCY.getVersion(),
|
||||
DEPENDENCY.getClassifier().orElse(null)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDependencyKeyIncludesArtifactId() {
|
||||
testDependencyKeyInclusion(
|
||||
Dependency.create(
|
||||
DEPENDENCY.getGroupId(),
|
||||
"xxx",
|
||||
DEPENDENCY.getVersion(),
|
||||
DEPENDENCY.getClassifier().orElse(null)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDependencyKeyIncludesVersion() {
|
||||
testDependencyKeyInclusion(
|
||||
Dependency.create(
|
||||
DEPENDENCY.getGroupId(),
|
||||
DEPENDENCY.getArtifactId(),
|
||||
"xxx",
|
||||
DEPENDENCY.getClassifier().orElse(null)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDependencyKeyIncludesClassifier() {
|
||||
testDependencyKeyInclusion(
|
||||
Dependency.create(
|
||||
DEPENDENCY.getGroupId(),
|
||||
DEPENDENCY.getArtifactId(),
|
||||
DEPENDENCY.getVersion(),
|
||||
"xxx"));
|
||||
}
|
||||
|
||||
private static void testDependencyKeyInclusion(Dependency modifiedDependency) {
|
||||
final DependencyTree dependencyTree = new DependencyTree();
|
||||
dependencyTree.addDirectDependency(DEPENDENCY);
|
||||
dependencyTree.addDirectDependency(modifiedDependency);
|
||||
|
||||
assertThat(dependencyTree.flatten()).containsExactly(DEPENDENCY, modifiedDependency);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDependencyKeyIgnoresScopeAndOptionalFlag() {
|
||||
final Dependency dependencyWithScopeAndOptionalFlag =
|
||||
Dependency.create(
|
||||
DEPENDENCY.getGroupId(),
|
||||
DEPENDENCY.getArtifactId(),
|
||||
DEPENDENCY.getVersion(),
|
||||
DEPENDENCY.getClassifier().orElse(null),
|
||||
"compile",
|
||||
true);
|
||||
|
||||
final DependencyTree dependencyTree = new DependencyTree();
|
||||
dependencyTree.addDirectDependency(DEPENDENCY);
|
||||
dependencyTree.addDirectDependency(dependencyWithScopeAndOptionalFlag);
|
||||
|
||||
assertThat(dependencyTree.flatten()).containsExactly(DEPENDENCY);
|
||||
assertThat(dependencyTree.getPathTo(dependencyWithScopeAndOptionalFlag))
|
||||
.containsExactly(DEPENDENCY);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.flink.util.TestLoggerExtension
|
||||
Loading…
Reference in New Issue