Refactored and implement of a template method pattern for logger config in webmagic-scripts (#1158)
* Refactor of processSingle in PageModelExtractor * Changed my refactor of processSingle, this one is a lot better * Changed my refactor of processSingle, this one is a lot better * add lombok for getters and setters * Refactored and implement of a template method pattern for logger configpull/1169/head
François Gibier
10 months ago
committed by
GitHub
parent
2df7dca871
commit
d8321baf56
@ -0,0 +1,47 @@
|
||||
package us.codecraft.webmagic.scripts;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import us.codecraft.webmagic.scripts.languages.JRuby;
|
||||
import us.codecraft.webmagic.scripts.languages.Javascript;
|
||||
import us.codecraft.webmagic.scripts.languages.Language;
|
||||
import us.codecraft.webmagic.utils.WMCollections;
|
||||
|
||||
public class Params {
|
||||
@Getter
|
||||
Language language = new Javascript();
|
||||
|
||||
@Getter @Setter
|
||||
String scriptFileName;
|
||||
|
||||
@Getter @Setter
|
||||
List<String> urls;
|
||||
|
||||
@Getter @Setter
|
||||
int thread = 1;
|
||||
|
||||
@Getter @Setter
|
||||
int sleepTime = 1000;
|
||||
|
||||
private static Map<Language, Set<String>> alias;
|
||||
|
||||
public Params() {
|
||||
alias = new HashMap<Language, Set<String>>();
|
||||
alias.put(new Javascript(), WMCollections.<String>newHashSet("js", "javascript", "JavaScript", "JS"));
|
||||
alias.put(new JRuby(), WMCollections.<String>newHashSet("ruby", "jruby", "Ruby", "JRuby"));
|
||||
}
|
||||
|
||||
public void setLanguagefromArg(String arg) {
|
||||
for (Map.Entry<Language, Set<String>> languageSetEntry : alias.entrySet()) {
|
||||
if (languageSetEntry.getValue().contains(arg)) {
|
||||
this.language = languageSetEntry.getKey();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,82 @@
|
||||
package us.codecraft.webmagic.scripts.config;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
|
||||
import lombok.Getter;
|
||||
import us.codecraft.webmagic.scripts.Params;
|
||||
|
||||
public abstract class CommandLineOption {
|
||||
@Getter
|
||||
char option;
|
||||
|
||||
public CommandLineOption(char option) {
|
||||
this.option = option;
|
||||
}
|
||||
|
||||
protected abstract void addParamOption(Params params, CommandLine commandLine);
|
||||
|
||||
public void addParamOptionIfInCommandLine(Params params, CommandLine commandLine) {
|
||||
if (commandLine.hasOption(this.option))
|
||||
this.addParamOption(params, commandLine);
|
||||
}
|
||||
|
||||
public static List<CommandLineOption> getAllOptions() {
|
||||
return List.of(new OptionL(), new OptionF(), new OptionS(), new OptionT(), new OptionG());
|
||||
}
|
||||
}
|
||||
|
||||
class OptionL extends CommandLineOption {
|
||||
public OptionL() {
|
||||
super('l');
|
||||
}
|
||||
|
||||
protected void addParamOption(Params params, CommandLine commandLine) {
|
||||
String language = commandLine.getOptionValue("l");
|
||||
params.setLanguagefromArg(language);
|
||||
}
|
||||
}
|
||||
|
||||
class OptionF extends CommandLineOption {
|
||||
public OptionF() {
|
||||
super('f');
|
||||
}
|
||||
|
||||
protected void addParamOption(Params params, CommandLine commandLine) {
|
||||
String scriptFilename = commandLine.getOptionValue("f");
|
||||
params.setScriptFileName(scriptFilename);
|
||||
}
|
||||
}
|
||||
|
||||
class OptionS extends CommandLineOption {
|
||||
public OptionS() {
|
||||
super('s');
|
||||
}
|
||||
|
||||
protected void addParamOption(Params params, CommandLine commandLine) {
|
||||
Integer sleepTime = Integer.parseInt(commandLine.getOptionValue("s"));
|
||||
params.setSleepTime(sleepTime);
|
||||
}
|
||||
}
|
||||
|
||||
class OptionT extends CommandLineOption {
|
||||
public OptionT() {
|
||||
super('t');
|
||||
}
|
||||
|
||||
protected void addParamOption(Params params, CommandLine commandLine) {
|
||||
Integer thread = Integer.parseInt(commandLine.getOptionValue("t"));
|
||||
params.setThread(thread);
|
||||
}
|
||||
}
|
||||
|
||||
class OptionG extends CommandLineOption {
|
||||
public OptionG() {
|
||||
super('g');
|
||||
}
|
||||
|
||||
protected void addParamOption(Params params, CommandLine commandLine) {
|
||||
ConfigLogger.configLogger(commandLine.getOptionValue("g"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
package us.codecraft.webmagic.scripts.config;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.core.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ConfigLogger {
|
||||
/**
|
||||
* Log the config parameter. If the counter is less than the number of available
|
||||
* options then it means that the user entered an option
|
||||
*
|
||||
* @param value The config string
|
||||
*/
|
||||
public static void configLogger(String value) {
|
||||
List<Pair<String, Level>> options = List.of(
|
||||
Pair.of("debug", Level.DEBUG),
|
||||
Pair.of("info", Level.INFO),
|
||||
Pair.of("warn", Level.WARN),
|
||||
Pair.of("trace", Level.TRACE),
|
||||
Pair.of("off", Level.OFF),
|
||||
Pair.of("error", Level.ERROR));
|
||||
Pair<String, Level> option = options.get(0);
|
||||
int i = 1;
|
||||
while (i < options.size() && !option.getLeft().equalsIgnoreCase(value))
|
||||
option = options.get(i++);
|
||||
if (i < options.size()) {
|
||||
Logger rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME);
|
||||
rootLogger.setLevel(option.getRight());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package us.codecraft.webmagic.scripts.languages;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.script.ScriptEngine;
|
||||
import javax.script.ScriptException;
|
||||
|
||||
import org.jruby.RubyHash;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
|
||||
public class JRuby extends Language {
|
||||
public JRuby() {
|
||||
super("jruby","ruby/defines.rb","");
|
||||
}
|
||||
|
||||
public void process(ScriptEngine engine, String defines, String script, Page page) throws ScriptException {
|
||||
RubyHash oRuby = (RubyHash) engine.eval(defines + "\n" + script, engine.getContext());
|
||||
Iterator itruby = oRuby.entrySet().iterator();
|
||||
while (itruby.hasNext()) {
|
||||
Map.Entry pairs = (Map.Entry) itruby.next();
|
||||
page.getResultItems().put(pairs.getKey().toString(), pairs.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package us.codecraft.webmagic.scripts.languages;
|
||||
|
||||
import javax.script.ScriptEngine;
|
||||
import javax.script.ScriptException;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
|
||||
public class Javascript extends Language {
|
||||
public Javascript() {
|
||||
super("javascript","js/defines.js","");
|
||||
}
|
||||
|
||||
public void process(ScriptEngine engine, String defines, String script, Page page) throws ScriptException {
|
||||
engine.eval(defines + "\n" + script, engine.getContext());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package us.codecraft.webmagic.scripts.languages;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.script.ScriptEngine;
|
||||
import javax.script.ScriptException;
|
||||
|
||||
import org.python.core.PyDictionary;
|
||||
|
||||
import us.codecraft.webmagic.Page;
|
||||
|
||||
public class Jython extends Language {
|
||||
public Jython() {
|
||||
super("jython","python/defines.py","");
|
||||
}
|
||||
|
||||
public void process(ScriptEngine engine, String defines, String script, Page page) throws ScriptException {
|
||||
engine.eval(defines + "\n" + script, engine.getContext());
|
||||
PyDictionary oJython = (PyDictionary) engine.get("result");
|
||||
Iterator it = oJython.entrySet().iterator();
|
||||
while (it.hasNext()) {
|
||||
Map.Entry pairs = (Map.Entry) it.next();
|
||||
page.getResultItems().put(pairs.getKey().toString(), pairs.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue