|
|
|
@ -1,21 +1,25 @@
|
|
|
|
|
package us.codecraft.webmagic.monitor;
|
|
|
|
|
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
import us.codecraft.webmagic.Request;
|
|
|
|
|
import us.codecraft.webmagic.Spider;
|
|
|
|
|
import us.codecraft.webmagic.SpiderListener;
|
|
|
|
|
import us.codecraft.webmagic.utils.Experimental;
|
|
|
|
|
import us.codecraft.webmagic.utils.UrlUtils;
|
|
|
|
|
|
|
|
|
|
import javax.management.*;
|
|
|
|
|
import java.lang.management.ManagementFactory;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Collections;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
|
|
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
|
|
|
|
|
import javax.management.InstanceAlreadyExistsException;
|
|
|
|
|
import javax.management.JMException;
|
|
|
|
|
import javax.management.MBeanRegistrationException;
|
|
|
|
|
import javax.management.MBeanServer;
|
|
|
|
|
import javax.management.MalformedObjectNameException;
|
|
|
|
|
import javax.management.NotCompliantMBeanException;
|
|
|
|
|
import javax.management.ObjectName;
|
|
|
|
|
|
|
|
|
|
import us.codecraft.webmagic.Request;
|
|
|
|
|
import us.codecraft.webmagic.Spider;
|
|
|
|
|
import us.codecraft.webmagic.SpiderListener;
|
|
|
|
|
import us.codecraft.webmagic.utils.Experimental;
|
|
|
|
|
import us.codecraft.webmagic.utils.UrlUtils;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @author code4crafer@gmail.com
|
|
|
|
|
* @since 0.5.0
|
|
|
|
@ -23,17 +27,13 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
|
@Experimental
|
|
|
|
|
public class SpiderMonitor {
|
|
|
|
|
|
|
|
|
|
private static SpiderMonitor INSTANCE = new SpiderMonitor();
|
|
|
|
|
|
|
|
|
|
private AtomicBoolean started = new AtomicBoolean(false);
|
|
|
|
|
|
|
|
|
|
private Logger logger = LoggerFactory.getLogger(getClass());
|
|
|
|
|
private static final SpiderMonitor INSTANCE = new SpiderMonitor();
|
|
|
|
|
|
|
|
|
|
private MBeanServer mbeanServer;
|
|
|
|
|
|
|
|
|
|
private String jmxServerName;
|
|
|
|
|
|
|
|
|
|
private List<SpiderStatusMXBean> spiderStatuses = new ArrayList<SpiderStatusMXBean>();
|
|
|
|
|
private List<SpiderStatusMXBean> spiderStatuses = new ArrayList<>();
|
|
|
|
|
|
|
|
|
|
protected SpiderMonitor() {
|
|
|
|
|
jmxServerName = "WebMagic";
|
|
|
|
@ -51,7 +51,7 @@ public class SpiderMonitor {
|
|
|
|
|
for (Spider spider : spiders) {
|
|
|
|
|
MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
|
|
|
|
|
if (spider.getSpiderListeners() == null) {
|
|
|
|
|
List<SpiderListener> spiderListeners = new ArrayList<SpiderListener>();
|
|
|
|
|
List<SpiderListener> spiderListeners = new ArrayList<>();
|
|
|
|
|
spiderListeners.add(monitorSpiderListener);
|
|
|
|
|
spider.setSpiderListeners(spiderListeners);
|
|
|
|
|
} else {
|
|
|
|
@ -90,7 +90,7 @@ public class SpiderMonitor {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void onError(Request request) {
|
|
|
|
|
public void onError(Request request, Exception e) {
|
|
|
|
|
errorUrls.add(request.getUrl());
|
|
|
|
|
errorCount.incrementAndGet();
|
|
|
|
|
}
|
|
|
|
@ -109,7 +109,6 @@ public class SpiderMonitor {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected void registerMBean(SpiderStatusMXBean spiderStatus) throws MalformedObjectNameException, InstanceAlreadyExistsException, MBeanRegistrationException, NotCompliantMBeanException {
|
|
|
|
|
// ObjectName objName = new ObjectName(jmxServerName + ":name=" + spiderStatus.getName());
|
|
|
|
|
ObjectName objName = new ObjectName(jmxServerName + ":name=" + UrlUtils.removePort(spiderStatus.getName()));
|
|
|
|
|
mbeanServer.registerMBean(spiderStatus, objName);
|
|
|
|
|
}
|
|
|
|
|