xpath2.0
yihua.huang
12 years ago
parent
7f27c28d4c
commit
7c9e9ce869
@ -0,0 +1,167 @@
|
|||||||
|
package us.codecraft.webmagic.selector;
|
||||||
|
|
||||||
|
import net.sf.saxon.lib.NamespaceConstant;
|
||||||
|
import net.sf.saxon.xpath.XPathEvaluator;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
import org.htmlcleaner.CleanerProperties;
|
||||||
|
import org.htmlcleaner.DomSerializer;
|
||||||
|
import org.htmlcleaner.HtmlCleaner;
|
||||||
|
import org.htmlcleaner.TagNode;
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.NodeList;
|
||||||
|
|
||||||
|
import javax.xml.namespace.NamespaceContext;
|
||||||
|
import javax.xml.transform.OutputKeys;
|
||||||
|
import javax.xml.transform.Transformer;
|
||||||
|
import javax.xml.transform.TransformerFactory;
|
||||||
|
import javax.xml.transform.dom.DOMSource;
|
||||||
|
import javax.xml.transform.stream.StreamResult;
|
||||||
|
import javax.xml.xpath.XPathConstants;
|
||||||
|
import javax.xml.xpath.XPathExpression;
|
||||||
|
import javax.xml.xpath.XPathExpressionException;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。<br>
|
||||||
|
*
|
||||||
|
* @author code4crafter@gmail.com <br>
|
||||||
|
* Date: 13-4-21
|
||||||
|
* Time: 上午9:39
|
||||||
|
*/
|
||||||
|
public class Xpath2Selector implements Selector {
|
||||||
|
|
||||||
|
private String xpathStr;
|
||||||
|
|
||||||
|
private XPathExpression xPathExpression;
|
||||||
|
|
||||||
|
private Logger logger = Logger.getLogger(getClass());
|
||||||
|
|
||||||
|
public Xpath2Selector(String xpathStr) {
|
||||||
|
this.xpathStr = xpathStr;
|
||||||
|
try {
|
||||||
|
init();
|
||||||
|
} catch (XPathExpressionException e) {
|
||||||
|
throw new IllegalArgumentException("XPath error!", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum XPath2NamespaceContext implements NamespaceContext {
|
||||||
|
|
||||||
|
INSTANCE;
|
||||||
|
|
||||||
|
private final Map<String, String> prefix2NamespaceMap = new ConcurrentHashMap<String, String>();
|
||||||
|
|
||||||
|
private final Map<String, List<String>> namespace2PrefixMap = new ConcurrentHashMap<String, List<String>>();
|
||||||
|
|
||||||
|
private void put(String prefix, String namespaceURI) {
|
||||||
|
prefix2NamespaceMap.put(prefix, namespaceURI);
|
||||||
|
List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
|
||||||
|
if (prefixes == null) {
|
||||||
|
prefixes = new ArrayList<String>();
|
||||||
|
namespace2PrefixMap.put(namespaceURI, prefixes);
|
||||||
|
}
|
||||||
|
prefixes.add(prefix);
|
||||||
|
}
|
||||||
|
|
||||||
|
private XPath2NamespaceContext() {
|
||||||
|
put("fn", NamespaceConstant.FN);
|
||||||
|
put("xslt",NamespaceConstant.XSLT);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getNamespaceURI(String prefix) {
|
||||||
|
return prefix2NamespaceMap.get(prefix);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getPrefix(String namespaceURI) {
|
||||||
|
List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
|
||||||
|
if (prefixes == null || prefixes.size() < 1) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return prefixes.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Iterator getPrefixes(String namespaceURI) {
|
||||||
|
List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
|
||||||
|
if (prefixes == null || prefixes.size() < 1) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return prefixes.iterator();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void init() throws XPathExpressionException {
|
||||||
|
XPathEvaluator xPathEvaluator = new XPathEvaluator();
|
||||||
|
xPathEvaluator.setNamespaceContext(XPath2NamespaceContext.INSTANCE);
|
||||||
|
xPathExpression = xPathEvaluator.compile(xpathStr);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String select(String text) {
|
||||||
|
try {
|
||||||
|
HtmlCleaner htmlCleaner = new HtmlCleaner();
|
||||||
|
TagNode tagNode = htmlCleaner.clean(text);
|
||||||
|
Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
|
||||||
|
Object result;
|
||||||
|
try {
|
||||||
|
result = xPathExpression.evaluate(document, XPathConstants.NODESET);
|
||||||
|
} catch (XPathExpressionException e) {
|
||||||
|
result = xPathExpression.evaluate(document, XPathConstants.STRING);
|
||||||
|
}
|
||||||
|
if (result instanceof NodeList) {
|
||||||
|
StreamResult xmlOutput = new StreamResult(new StringWriter());
|
||||||
|
Transformer transformer = TransformerFactory.newInstance().newTransformer();
|
||||||
|
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
|
||||||
|
NodeList nodeList = (NodeList) result;
|
||||||
|
if (nodeList.getLength() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
transformer.transform(new DOMSource(nodeList.item(0)), xmlOutput);
|
||||||
|
return xmlOutput.getWriter().toString();
|
||||||
|
}
|
||||||
|
return result.toString();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("select text error! " + xpathStr, e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<String> selectList(String text) {
|
||||||
|
List<String> results = new ArrayList<String>();
|
||||||
|
try {
|
||||||
|
HtmlCleaner htmlCleaner = new HtmlCleaner();
|
||||||
|
TagNode tagNode = htmlCleaner.clean(text);
|
||||||
|
Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
|
||||||
|
Object result;
|
||||||
|
try {
|
||||||
|
result = xPathExpression.evaluate(document, XPathConstants.NODESET);
|
||||||
|
} catch (XPathExpressionException e) {
|
||||||
|
result = xPathExpression.evaluate(document, XPathConstants.STRING);
|
||||||
|
}
|
||||||
|
if (result instanceof NodeList) {
|
||||||
|
NodeList nodeList = (NodeList) result;
|
||||||
|
Transformer transformer = TransformerFactory.newInstance().newTransformer();
|
||||||
|
StreamResult xmlOutput = new StreamResult();
|
||||||
|
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
|
||||||
|
for (int i = 0; i < nodeList.getLength(); i++) {
|
||||||
|
xmlOutput.setWriter(new StringWriter());
|
||||||
|
transformer.transform(new DOMSource(nodeList.item(i)), xmlOutput);
|
||||||
|
results.add(xmlOutput.getWriter().toString());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
results.add(result.toString());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("select text error! " + xpathStr, e);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue