refactor in selectors
yihua.huang
12 years ago
parent
85b7cf1563
commit
2c3574537a
@ -1,91 +0,0 @@
|
||||
package us.codecraft.webmagic.selector;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Selector factory with some inner cache.<br>
|
||||
*
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @since 0.1.0
|
||||
*/
|
||||
public class SelectorFactory {
|
||||
|
||||
private Map<String, Selector> innerCache = new ConcurrentHashMap<String, Selector>();
|
||||
|
||||
private static final SelectorFactory INSTATNCE = new SelectorFactory();
|
||||
|
||||
public static SelectorFactory getInstatnce() {
|
||||
return INSTATNCE;
|
||||
}
|
||||
|
||||
public RegexSelector newRegexSelector(String regex) {
|
||||
return newSelector(RegexSelector.class, regex);
|
||||
}
|
||||
|
||||
public RegexSelector newRegexSelector(String regex, int group) {
|
||||
String cacheKey = getCacheKey(RegexSelector.class, regex, String.valueOf(group));
|
||||
if (innerCache.get(cacheKey) != null) {
|
||||
return (RegexSelector) innerCache.get(cacheKey);
|
||||
}
|
||||
return new RegexSelector(regex, group);
|
||||
}
|
||||
|
||||
public ReplaceSelector newReplaceSelector(String regex, String replacement) {
|
||||
return newSelector(ReplaceSelector.class, regex, replacement);
|
||||
}
|
||||
|
||||
public XpathSelector newXpathSelector(String xpath) {
|
||||
return newSelector(XpathSelector.class, xpath);
|
||||
}
|
||||
|
||||
public SmartContentSelector newSmartContentSelector() {
|
||||
return newSelector(SmartContentSelector.class);
|
||||
}
|
||||
|
||||
public <T extends Selector> T newAndCacheSelector(Class<T> clazz, String... param) {
|
||||
String cacheKey = getCacheKey(RegexSelector.class, param);
|
||||
if (innerCache.get(cacheKey) != null) {
|
||||
return (T) innerCache.get(cacheKey);
|
||||
}
|
||||
T selector = newSelector(clazz, param);
|
||||
if (selector != null) {
|
||||
innerCache.put(cacheKey, selector);
|
||||
}
|
||||
return selector;
|
||||
|
||||
}
|
||||
|
||||
public <T extends Selector> T newSelector(Class<T> clazz, String... param) {
|
||||
try {
|
||||
if (param.length == 0) {
|
||||
Constructor<T> constructor
|
||||
= clazz.getConstructor();
|
||||
T selector = constructor.newInstance();
|
||||
return selector;
|
||||
} else if (param.length == 1) {
|
||||
Constructor<T> constructor
|
||||
= clazz.getConstructor(String.class);
|
||||
T selector = constructor.newInstance(param[0]);
|
||||
return selector;
|
||||
} else if (param.length == 2) {
|
||||
Constructor<T> constructor
|
||||
= clazz.getConstructor(String.class, String.class);
|
||||
T selector = constructor.newInstance(param[0], param[1]);
|
||||
return selector;
|
||||
} else {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException("init object error", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String getCacheKey(Class<?> clazz, String... param) {
|
||||
return clazz.toString() + "_" + StringUtils.join(param, "_");
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
package us.codecraft.webmagic.selector;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Extract text content in html.<br>
|
||||
* Algorithm from <a href="http://www.elias.cn/En/ExtMainText">http://www.elias.cn/En/ExtMainText</a>. <br>
|
||||
*
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @since 0.2.2
|
||||
*/
|
||||
public class TextContentSelector implements Selector {
|
||||
|
||||
private String newLineSeperator = "\n";
|
||||
|
||||
public TextContentSelector() {
|
||||
}
|
||||
|
||||
public TextContentSelector(String newLineSeperator) {
|
||||
this.newLineSeperator = newLineSeperator;
|
||||
}
|
||||
|
||||
private final static Set<String> TAGS_IN_NEWLINE = new HashSet<String>();
|
||||
|
||||
private final static Set<String> TAGS_TO_IGNORE = new HashSet<String>();
|
||||
|
||||
static {
|
||||
TAGS_IN_NEWLINE.addAll(Arrays.asList(new String[]{"p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "br", "li"}));
|
||||
TAGS_TO_IGNORE.addAll(Arrays.asList(new String[]{"head", "style", "script", "noscript", "option"}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String select(String text) {
|
||||
Document doc = Jsoup.parse(text);
|
||||
return select0(doc);
|
||||
}
|
||||
|
||||
protected String select0(Element element) {
|
||||
String tagName = element.tagName().toLowerCase();
|
||||
if (TAGS_TO_IGNORE.contains(tagName)) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder textBuilder = new StringBuilder();
|
||||
textBuilder.append(element.text());
|
||||
if (element.children() != null) {
|
||||
for (Element child : element.children()) {
|
||||
textBuilder.append(select0(child));
|
||||
}
|
||||
}
|
||||
if (TAGS_IN_NEWLINE.contains(tagName)) {
|
||||
textBuilder.append(newLineSeperator);
|
||||
}
|
||||
return textBuilder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> selectList(String text) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package us.codecraft.webmagic.selector;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import us.codecraft.webmagic.downloader.HttpClientDownloader;
|
||||
|
||||
/**
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* @since 0.2.2
|
||||
*/
|
||||
public class TextContentSelectorTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
String html = "<div class=\"edit-comment-hide\">\n" +
|
||||
" <div class=\"js-comment-body comment-body markdown-body markdown-format\">\n" +
|
||||
" <p>Add more powerful selector for content text extract refered to <a href=\"http://www.elias.cn/En/ExtMainText\">http://www.elias.cn/En/ExtMainText</a></p>\n" +
|
||||
" </div>\n" +
|
||||
" </div>";
|
||||
TextContentSelector textContentSelector = new TextContentSelector("<br>");
|
||||
String text = textContentSelector.select(html);
|
||||
Assert.assertNotNull(text);
|
||||
}
|
||||
|
||||
@Ignore("takes long time")
|
||||
@Test
|
||||
public void testDownload() {
|
||||
String s = new HttpClientDownloader().download("http://blog.codecraft.us/blog/2013/08/18/ti-yan-dao-liao-open-sourcede-mei-li/", "utf-8")
|
||||
.smartContent().text().toString();
|
||||
Assert.assertNotNull(s);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue