From 2edd2389cd66225c95723789d49d170d4b3b0d13 Mon Sep 17 00:00:00 2001 From: hengyunabc Date: Mon, 10 Aug 2020 20:27:25 +0800 Subject: [PATCH] update download link to arthas.aliyun.com. #1411 --- bin/as.sh | 26 +--- .../com/taobao/arthas/boot/Bootstrap.java | 36 ++---- .../com/taobao/arthas/boot/DownloadUtils.java | 119 +++++------------- .../taobao/arthas/boot/DownloadUtilsTest.java | 71 +++++------ 4 files changed, 78 insertions(+), 174 deletions(-) diff --git a/bin/as.sh b/bin/as.sh index 43a75385b..11047f855 100755 --- a/bin/as.sh +++ b/bin/as.sh @@ -103,7 +103,7 @@ SESSION_TIMEOUT=1800 # use specify version USE_VERSION= -# maven repo to download arthas +# remote repo to download arthas REPO_MIRROR= # use http to download arthas @@ -150,13 +150,9 @@ TMP_DIR=/tmp # last update arthas version ARTHAS_VERSION= -# maven-metadata.xml url -# https://repo1.maven.org/maven2/com/taobao/arthas/arthas-packaging/maven-metadata.xml -MAVEN_METADATA_URL="PLACEHOLDER_REPO/com/taobao/arthas/arthas-packaging/maven-metadata.xml" # arthas remote url -# https://repo1.maven.org/maven2/com/taobao/arthas/arthas-packaging/3.x.x/arthas-packaging-3.x.x-bin.zip -REMOTE_DOWNLOAD_URL="PLACEHOLDER_REPO/com/taobao/arthas/arthas-packaging/PLACEHOLDER_VERSION/arthas-packaging-PLACEHOLDER_VERSION-bin.zip" - +# https://arthas.aliyun.com/download/3.1.7?mirror=aliyun +REMOTE_DOWNLOAD_URL="https://arthas.aliyun.com/download/PLACEHOLDER_VERSION?mirror=PLACEHOLDER_REPO" # update timeout(sec) SO_TIMEOUT=5 @@ -314,16 +310,7 @@ get_local_version() get_repo_url() { - local repoUrl="" - if [[ $REPO_MIRROR == "center" ]] ; then - repoUrl="https://repo1.maven.org/maven2" - fi - if [[ $REPO_MIRROR == "aliyun" ]] ; then - repoUrl="https://maven.aliyun.com/repository/public" - fi - if [ -z ${repoUrl} ] ; then - repoUrl="${REPO_MIRROR}" - fi + local repoUrl="${REPO_MIRROR}" if [ "$USE_HTTP" = true ] ; then repoUrl=${repoUrl/https/http} fi @@ -333,8 +320,7 @@ get_repo_url() # get latest version from remote get_remote_version() { - local url="${MAVEN_METADATA_URL//PLACEHOLDER_REPO/$(get_repo_url)}" - curl -sLk "${url}" | sed -n -e 's/.*\(.*\)<\/release>.*/\1/p' | head -n 1 + curl -sLk "https://arthas.aliyun.com/api/latest_version" } # check version greater @@ -418,7 +404,7 @@ Options and Arguments: --session-timeout The session timeout seconds, default 1800 (30min) --arthas-home The arthas home --use-version Use special version arthas - --repo-mirror Use special maven repository mirror, value is + --repo-mirror Use special remote repository mirror, value is center/aliyun or http repo url. --versions List local and remote arthas versions --use-http Enforce use http to download, default use https diff --git a/boot/src/main/java/com/taobao/arthas/boot/Bootstrap.java b/boot/src/main/java/com/taobao/arthas/boot/Bootstrap.java index 1891cd98a..75d38a69b 100644 --- a/boot/src/main/java/com/taobao/arthas/boot/Bootstrap.java +++ b/boot/src/main/java/com/taobao/arthas/boot/Bootstrap.java @@ -85,7 +85,7 @@ public class Bootstrap { * The directory contains arthas-core.jar/arthas-client.jar/arthas-spy.jar. * 1. When use-version is not empty, try to find arthas home under ~/.arthas/lib * 2. Try set the directory where arthas-boot.jar is located to arthas home - * 3. Try to download from maven repo + * 3. Try to download from remote repo * */ private String arthasHome; @@ -101,7 +101,7 @@ public class Bootstrap { private boolean versions; /** - * download from maven repository. if timezone is +0800, default value is 'aliyun', else is 'center'. + * download from remo repository. if timezone is +0800, default value is 'aliyun', else is 'center'. */ private String repoMirror; @@ -193,7 +193,7 @@ public class Bootstrap { } @Option(longName = "repo-mirror") - @Description("Use special maven repository mirror, value is center/aliyun or http repo url.") + @Description("Use special remote repository mirror, value is center/aliyun or http repo url.") public void setRepoMirror(String repoMirror) { this.repoMirror = repoMirror; } @@ -281,8 +281,6 @@ public class Bootstrap { } } - String mavenMetaData = null; - Bootstrap bootstrap = new Bootstrap(); CLI cli = CLIConfigurator.define(Bootstrap.class); @@ -314,10 +312,7 @@ public class Bootstrap { AnsiLog.debug("Repo mirror:" + bootstrap.getRepoMirror()); if (bootstrap.isVersions()) { - if (mavenMetaData == null) { - mavenMetaData = DownloadUtils.readMavenMetaData(bootstrap.getRepoMirror(), bootstrap.isuseHttp()); - } - System.out.println(UsageRender.render(listVersions(mavenMetaData))); + System.out.println(UsageRender.render(listVersions())); System.exit(0); } @@ -427,17 +422,13 @@ public class Bootstrap { localLastestVersion = versionList.get(versionList.size() - 1); } - if (mavenMetaData == null) { - mavenMetaData = DownloadUtils.readMavenMetaData(bootstrap.getRepoMirror(), bootstrap.isuseHttp()); - } - - String remoteLastestVersion = DownloadUtils.readMavenReleaseVersion(mavenMetaData); + String remoteLastestVersion = DownloadUtils.readLatestReleaseVersion(); boolean needDownload = false; if (localLastestVersion == null) { if (remoteLastestVersion == null) { // exit - AnsiLog.error("Can not find Arthas under local: {} and remote maven repo mirror: {}", ARTHAS_LIB_DIR, + AnsiLog.error("Can not find Arthas under local: {} and remote repo mirror: {}", ARTHAS_LIB_DIR, bootstrap.getRepoMirror()); AnsiLog.error( "Unable to download arthas from remote server, please download the full package according to wiki: https://github.com/alibaba/arthas"); @@ -630,7 +621,7 @@ public class Bootstrap { return -1; } - private static String listVersions(String mavenMetaData) { + private static String listVersions() { StringBuilder result = new StringBuilder(1024); List versionList = listNames(ARTHAS_LIB_DIR); Collections.sort(versionList); @@ -640,13 +631,12 @@ public class Bootstrap { result.append(" " + version).append('\n'); } result.append("Remote versions:\n"); - if (mavenMetaData != null) { - List remoteVersions = DownloadUtils.readAllMavenVersion(mavenMetaData); - Collections.reverse(remoteVersions); - for (String version : remoteVersions) { - result.append(" " + version).append('\n'); - } - } + + List remoteVersions = DownloadUtils.readRemoteVersions(); + Collections.reverse(remoteVersions); + for (String version : remoteVersions) { + result.append(" " + version).append('\n'); + } return result.toString(); } diff --git a/boot/src/main/java/com/taobao/arthas/boot/DownloadUtils.java b/boot/src/main/java/com/taobao/arthas/boot/DownloadUtils.java index 750de95b5..9a37fcc57 100644 --- a/boot/src/main/java/com/taobao/arthas/boot/DownloadUtils.java +++ b/boot/src/main/java/com/taobao/arthas/boot/DownloadUtils.java @@ -1,7 +1,6 @@ package com.taobao.arthas.boot; import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -14,12 +13,6 @@ import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; - import com.taobao.arthas.common.AnsiLog; import com.taobao.arthas.common.IOUtils; @@ -29,64 +22,44 @@ import com.taobao.arthas.common.IOUtils; * */ public class DownloadUtils { - private static final String MAVEN_METADATA_URL = "${REPO}/com/taobao/arthas/arthas-packaging/maven-metadata.xml"; - private static final String REMOTE_DOWNLOAD_URL = "${REPO}/com/taobao/arthas/arthas-packaging/${VERSION}/arthas-packaging-${VERSION}-bin.zip"; + private static final String ARTHAS_VERSIONS_URL = "https://arthas.aliyun.com/api/versions"; + private static final String ARTHAS_LATEST_VERSIONS_URL = "https://arthas.aliyun.com/api/latest_version"; + + private static final String ARTHAS_DOWNLOAD_URL = "https://arthas.aliyun.com/download/${VERSION}?mirror=${REPO}"; private static final int CONNECTION_TIMEOUT = 3000; - /** - * Read release version from maven-metadata.xml - * - * @param mavenMetaData - * @return - */ - public static String readMavenReleaseVersion(String mavenMetaData) { + public static String readLatestReleaseVersion() { + InputStream inputStream = null; try { - Document document = transformMavenMetaData(mavenMetaData); - NodeList nodeList = document.getDocumentElement().getElementsByTagName("release"); - return nodeList.item(0).getTextContent(); - } catch (Exception e) { - // ignore + URLConnection connection = openURLConnection(ARTHAS_LATEST_VERSIONS_URL); + inputStream = connection.getInputStream(); + return IOUtils.toString(inputStream).trim(); + } catch (Throwable t) { + AnsiLog.error("Can not read arthas version from: " + ARTHAS_LATEST_VERSIONS_URL); + AnsiLog.debug(t); + } finally { + IOUtils.close(inputStream); } return null; } - /** - * Read all versions from maven-metadata.xml - * - * @param mavenMetaData - * @return - */ - public static List readAllMavenVersion(String mavenMetaData) { - List result = new ArrayList(); - try { - Document document = transformMavenMetaData(mavenMetaData); - NodeList nodeList = document.getDocumentElement().getElementsByTagName("version"); - int length = nodeList.getLength(); - for (int i = 0; i < length; ++i) { - result.add(nodeList.item(i).getTextContent()); - } - } catch (Exception e) { - // ignore - } - return result; - } - - public static String readMavenMetaData(String repoMirror, boolean http) { - String repoUrl = getRepoUrl(repoMirror, http); - String metaDataUrl = MAVEN_METADATA_URL.replace("${REPO}", repoUrl); - AnsiLog.debug("Download maven-metadata.xml from: {}", metaDataUrl); + public static List readRemoteVersions() { InputStream inputStream = null; try { - URLConnection connection = openURLConnection(metaDataUrl); + URLConnection connection = openURLConnection(ARTHAS_VERSIONS_URL); inputStream = connection.getInputStream(); - return IOUtils.toString(inputStream); - } catch (javax.net.ssl.SSLException e) { - AnsiLog.error("TLS connect error, please try to use --use-http argument."); - AnsiLog.error("URL: " + metaDataUrl); - AnsiLog.error(e); + String versionsStr = IOUtils.toString(inputStream); + String[] versions = versionsStr.split("\r\n"); + + ArrayList result = new ArrayList(); + for (String version : versions) { + result.add(version.trim()); + } + return result; + } catch (Throwable t) { - AnsiLog.error("Can not read maven-metadata.xml from: " + metaDataUrl); + AnsiLog.error("Can not read arthas versions from: " + ARTHAS_VERSIONS_URL); AnsiLog.debug(t); } finally { IOUtils.close(inputStream); @@ -94,16 +67,7 @@ public class DownloadUtils { return null; } - public static String getRepoUrl(String repoMirror, boolean http) { - repoMirror = repoMirror.trim(); - String repoUrl; - if (repoMirror.equals("center")) { - repoUrl = "https://repo1.maven.org/maven2"; - } else if (repoMirror.equals("aliyun")) { - repoUrl = "https://maven.aliyun.com/repository/public"; - } else { - repoUrl = repoMirror; - } + private static String getRepoUrl(String repoUrl, boolean http) { if (repoUrl.endsWith("/")) { repoUrl = repoUrl.substring(0, repoUrl.length() - 1); } @@ -116,7 +80,7 @@ public class DownloadUtils { public static void downArthasPackaging(String repoMirror, boolean http, String arthasVersion, String savePath) throws IOException { - String repoUrl = getRepoUrl(repoMirror, http); + String repoUrl = getRepoUrl(ARTHAS_DOWNLOAD_URL, http); File unzipDir = new File(savePath, arthasVersion + File.separator + "arthas"); @@ -124,14 +88,14 @@ public class DownloadUtils { AnsiLog.debug("Arthas download temp file: " + tempFile.getAbsolutePath()); - String remoteDownloadUrl = REMOTE_DOWNLOAD_URL.replace("${REPO}", repoUrl).replace("${VERSION}", arthasVersion); + String remoteDownloadUrl = repoUrl.replace("${REPO}", repoMirror).replace("${VERSION}", arthasVersion); AnsiLog.info("Start download arthas from remote server: " + remoteDownloadUrl); saveUrl(tempFile.getAbsolutePath(), remoteDownloadUrl, true); AnsiLog.info("Download arthas success."); IOUtils.unzip(tempFile.getAbsolutePath(), unzipDir.getAbsolutePath()); } - public static void saveUrl(final String filename, final String urlString, boolean printProgress) + private static void saveUrl(final String filename, final String urlString, boolean printProgress) throws IOException { BufferedInputStream in = null; FileOutputStream fout = null; @@ -176,29 +140,6 @@ public class DownloadUtils { } } - /** - * transform the maven meta data which is in the format of String into document object. - * - * @param mavenMetaData - * @return - * @throws Exception - */ - static Document transformMavenMetaData(String mavenMetaData) throws Exception { - ByteArrayInputStream inputStream = new ByteArrayInputStream(mavenMetaData.getBytes("UTF-8")); - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - //disable XXE before newDocumentBuilder - dbFactory.setFeature("http://javax.xml.XMLConstants/feature/secure-processing", true); - dbFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - dbFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - dbFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); - dbFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - dbFactory.setXIncludeAware(false); - dbFactory.setExpandEntityReferences(false); - //create doc builder - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - return dBuilder.parse(inputStream); - } - /** * support redirect * diff --git a/boot/src/test/java/com/taobao/arthas/boot/DownloadUtilsTest.java b/boot/src/test/java/com/taobao/arthas/boot/DownloadUtilsTest.java index d30e10632..a681ef112 100644 --- a/boot/src/test/java/com/taobao/arthas/boot/DownloadUtilsTest.java +++ b/boot/src/test/java/com/taobao/arthas/boot/DownloadUtilsTest.java @@ -1,64 +1,51 @@ package com.taobao.arthas.boot; -import com.taobao.arthas.common.IOUtils; - -import org.junit.Test; - +import java.io.File; import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; +import java.util.List; import org.junit.Assert; -import org.w3c.dom.Document; - -import static com.taobao.arthas.boot.DownloadUtils.*; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; public class DownloadUtilsTest { + @Rule + public TemporaryFolder rootFolder = new TemporaryFolder(); @Test - public void testReadMavenReleaseVersion() { - //check 'center' repo - String releaseVersion = readMavenReleaseVersion(readMavenMetaData("center", false)); + public void testReadReleaseVersion() { + String releaseVersion = DownloadUtils.readLatestReleaseVersion(); Assert.assertNotNull(releaseVersion); Assert.assertNotEquals("releaseVersion is empty", "", releaseVersion.trim()); - //check 'aliyun' repo - String aliyunReleaseVersion = readMavenReleaseVersion(readMavenMetaData("aliyun", false)); - Assert.assertEquals("releaseVersion is not match between repo 'center' and 'aliyun'", releaseVersion, aliyunReleaseVersion); - } - - @Test - public void testReadAllMavenVersion() { - Assert.assertNotEquals(new ArrayList(), readAllMavenVersion(readMavenMetaData("center", false))); + System.err.println(releaseVersion); } @Test - public void testGetRepoUrl() { - Assert.assertEquals("http", getRepoUrl("https/", true)); - Assert.assertEquals("https://repo1.maven.org/maven2", getRepoUrl("center", false)); - Assert.assertEquals("https://maven.aliyun.com/repository/public", getRepoUrl("aliyun", false)); + public void testReadAllVersions() { + List versions = DownloadUtils.readRemoteVersions(); + Assert.assertEquals("", true, versions.contains("3.1.7")); } @Test - public void testReadMavenMetaData() throws IOException { - String url = "https://repo1.maven.org/maven2/com/taobao/arthas/arthas-packaging/maven-metadata.xml"; - Assert.assertEquals(IOUtils.toString(new URL(url).openStream()), readMavenMetaData("center", false)); + public void testAliyunDownload() throws IOException { + String version = "3.3.7"; + File folder = rootFolder.newFolder(); + System.err.println(folder.getAbsolutePath()); + DownloadUtils.downArthasPackaging("aliyun", false, version, folder.getAbsolutePath()); + + File as = new File(folder, version + File.separator + "arthas" + File.separator + "as.sh"); + Assert.assertTrue(as.exists()); } @Test - public void testXXE() throws Exception { - try { - //from https://blog.spoock.com/2018/10/23/java-xxe/ - String playload = "\n" + - "\n" + - " ]>\n" + - "&xxe;"; - Document document = transformMavenMetaData(playload); - } catch (org.xml.sax.SAXParseException e) { - String message = e.getMessage(); - Assert.assertTrue("XXE is not disabled", message.contains("disallow-doctype-decl")); - return; - } - Assert.fail("XXE is not disabled"); + public void testCenterDownload() throws IOException { + String version = "3.1.7"; + File folder = rootFolder.newFolder(); + System.err.println(folder.getAbsolutePath()); + DownloadUtils.downArthasPackaging("center", false, version, folder.getAbsolutePath()); + + File as = new File(folder, version + File.separator + "arthas" + File.separator + "as.sh"); + Assert.assertTrue(as.exists()); } }