From d2b2eed9df619d85d9cb6d808f7ef4bdc0c50d5f Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Wed, 19 Oct 2022 22:10:04 +0800 Subject: [PATCH] Pass the task to onSuccess & onError. --- .../webmagic/downloader/AbstractDownloader.java | 16 ++++++++++++++-- .../downloader/HttpClientDownloader.java | 4 ++-- .../webmagic/downloader/PhantomJSDownloader.java | 4 ++-- .../downloader/selenium/SeleniumDownloader.java | 4 ++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java index eb3a3a35..ea3bbc59 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java @@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Task; import us.codecraft.webmagic.selector.Html; /** @@ -35,15 +36,26 @@ public abstract class AbstractDownloader implements Downloader { return (Html) page.getHtml(); } + @Deprecated protected void onSuccess(Request request) { } + /** + * @since 0.7.6 + */ + protected void onSuccess(Request request, Task task) { + this.onSuccess(request); + } + @Deprecated protected void onError(Request request) { - this.onError(request, null); } - protected void onError(Request request, Throwable e) { + /** + * @since 0.7.6 + */ + protected void onError(Request request, Task task, Throwable e) { + this.onError(request); } } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 89b60389..f138b200 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -82,12 +82,12 @@ public class HttpClientDownloader extends AbstractDownloader { try { httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext()); page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); - onSuccess(request); + onSuccess(request, task); logger.info("downloading page success {}", request.getUrl()); return page; } catch (IOException e) { logger.warn("download page {} error", request.getUrl(), e); - onError(request, e); + onError(request, task, e); return page; } finally { if (httpResponse != null) { diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java index 88b8237e..4f1eee8e 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java @@ -98,9 +98,9 @@ public class PhantomJSDownloader extends AbstractDownloader { page.setRequest(request); page.setStatusCode(200); } - onSuccess(request); + onSuccess(request, task); } catch (Exception e) { - onError(request, e); + onError(request, task, e); logger.warn("download page {} error", request.getUrl(), e); } return page; diff --git a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java index df601b4f..39b3bc91 100644 --- a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java +++ b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java @@ -111,10 +111,10 @@ public class SeleniumDownloader extends AbstractDownloader implements Closeable page.setHtml(new Html(content, request.getUrl())); page.setUrl(new PlainText(request.getUrl())); page.setRequest(request); - onSuccess(request); + onSuccess(request, task); } catch (Exception e) { logger.warn("download page {} error", request.getUrl(), e); - onError(request, e); + onError(request, task, e); } finally { if (webDriver != null) { webDriverPool.returnToPool(webDriver);