MapReduce implementation. #312
Nikita
8 years ago
parent
5edf70a450
commit
5d83a86d33
@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Collates result from {@link RReducer} tasks and produces a single result object.
|
||||
* Executes only once.
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <K> key type
|
||||
* @param <V> value type
|
||||
* @param <R> result type
|
||||
*/
|
||||
public interface RCollator<K, V, R> extends Serializable {
|
||||
|
||||
/**
|
||||
* Collates result map from reduce phase of MapReduce process.
|
||||
*
|
||||
* @param resultMap contains reduced entires
|
||||
* @return single result object
|
||||
*/
|
||||
R collate(Map<K, V> resultMap);
|
||||
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
import org.redisson.api.RList;
|
||||
import org.redisson.api.RScoredSortedSet;
|
||||
import org.redisson.api.RSet;
|
||||
import org.redisson.api.RSetCache;
|
||||
import org.redisson.api.RSortedSet;
|
||||
|
||||
/**
|
||||
*
|
||||
* MapReduce allows to process large amount of data stored in
|
||||
* {@link RSet}, {@link RList}, {@link RSetCache}, {@link RScoredSortedSet}, {@link RSortedSet} and others
|
||||
* using Mapper, Reducer and/or Collator tasks launched across Redisson Nodes.
|
||||
* <p>
|
||||
* Usage example:
|
||||
*
|
||||
* <pre>
|
||||
* public class WordMapper implements RCollectionMapper<String, String, Integer> {
|
||||
*
|
||||
* public void map(String value, RCollector<String, Integer> collector) {
|
||||
* String[] words = value.split("[^a-zA-Z]");
|
||||
* for (String word : words) {
|
||||
* collector.emit(word, 1);
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* public class WordReducer implements RReducer<String, Integer> {
|
||||
*
|
||||
* public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
* int sum = 0;
|
||||
* while (iter.hasNext()) {
|
||||
* Integer i = (Integer) iter.next();
|
||||
* sum += i;
|
||||
* }
|
||||
* return sum;
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* public class WordCollator implements RCollator<String, Integer, Integer> {
|
||||
*
|
||||
* public Integer collate(Map<String, Integer> resultMap) {
|
||||
* int result = 0;
|
||||
* for (Integer count : resultMap.values()) {
|
||||
* result += count;
|
||||
* }
|
||||
* return result;
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* RList<String> list = redisson.getList("myWords");
|
||||
*
|
||||
* Map<String, Integer> wordsCount = list.<String, Integer>mapReduce()
|
||||
* .mapper(new WordMapper())
|
||||
* .reducer(new WordReducer())
|
||||
* .execute();
|
||||
*
|
||||
* Integer totalCount = list.<String, Integer>mapReduce()
|
||||
* .mapper(new WordMapper())
|
||||
* .reducer(new WordReducer())
|
||||
* .execute(new WordCollator());
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <VIn> input value
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public interface RCollectionMapReduce<VIn, KOut, VOut> extends RMapReduceExecutor<VIn, KOut, VOut> {
|
||||
|
||||
/**
|
||||
* Setup Mapper object
|
||||
*
|
||||
* @param mapper used during MapReduce
|
||||
* @return self instance
|
||||
*/
|
||||
RCollectionMapReduce<VIn, KOut, VOut> mapper(RCollectionMapper<VIn, KOut, VOut> mapper);
|
||||
|
||||
/**
|
||||
* Setup Reducer object
|
||||
*
|
||||
* @param reducer used during MapReduce
|
||||
* @return self instance
|
||||
*/
|
||||
RCollectionMapReduce<VIn, KOut, VOut> reducer(RReducer<KOut, VOut> reducer);
|
||||
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
/**
|
||||
* Mapper task invoked during map phase of MapReduce process and launched across Redisson Nodes.
|
||||
* Every task stores transformed result of input key and value into {@link RCollector} instance.
|
||||
* Collected results are handled by {@link RReducer} instance once
|
||||
* all Mapper tasks have finished.
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <VIn> input value
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public interface RCollectionMapper<VIn, KOut, VOut> {
|
||||
|
||||
/**
|
||||
* Invoked for each Collection source entry
|
||||
*
|
||||
* @param value - input value
|
||||
* @param collector - instance shared across all Mapper tasks
|
||||
*/
|
||||
void map(VIn value, RCollector<KOut, VOut> collector);
|
||||
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
/**
|
||||
* Stores each key/value mapping during map phase of MapReduce process.
|
||||
* Later used in reduce phase.
|
||||
*
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <K> key type
|
||||
* @param <V> value type
|
||||
*/
|
||||
public interface RCollector<K, V> {
|
||||
|
||||
/**
|
||||
* Store key/value
|
||||
*
|
||||
* @param key available to reduce
|
||||
* @param value available to reduce
|
||||
*/
|
||||
void emit(K key, V value);
|
||||
|
||||
}
|
||||
@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
/**
|
||||
*
|
||||
* MapReduce allows to process large amount of data stored in Redis map
|
||||
* using Mapper, Reducer and/or Collator tasks launched across Redisson Nodes.
|
||||
* <p>
|
||||
* Usage example:
|
||||
*
|
||||
* <pre>
|
||||
* public class WordMapper implements RMapper<String, String, String, Integer> {
|
||||
*
|
||||
* public void map(String key, String value, RCollector<String, Integer> collector) {
|
||||
* String[] words = value.split("[^a-zA-Z]");
|
||||
* for (String word : words) {
|
||||
* collector.emit(word, 1);
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* public class WordReducer implements RReducer<String, Integer> {
|
||||
*
|
||||
* public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
* int sum = 0;
|
||||
* while (iter.hasNext()) {
|
||||
* Integer i = (Integer) iter.next();
|
||||
* sum += i;
|
||||
* }
|
||||
* return sum;
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* public class WordCollator implements RCollator<String, Integer, Integer> {
|
||||
*
|
||||
* public Integer collate(Map<String, Integer> resultMap) {
|
||||
* int result = 0;
|
||||
* for (Integer count : resultMap.values()) {
|
||||
* result += count;
|
||||
* }
|
||||
* return result;
|
||||
* }
|
||||
*
|
||||
* }
|
||||
*
|
||||
* RMap<String, String> map = redisson.getMap("myWords");
|
||||
*
|
||||
* Map<String, Integer> wordsCount = map.<String, Integer>mapReduce()
|
||||
* .mapper(new WordMapper())
|
||||
* .reducer(new WordReducer())
|
||||
* .execute();
|
||||
*
|
||||
* Integer totalCount = map.<String, Integer>mapReduce()
|
||||
* .mapper(new WordMapper())
|
||||
* .reducer(new WordReducer())
|
||||
* .execute(new WordCollator());
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KIn> input key
|
||||
* @param <VIn> input value
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public interface RMapReduce<KIn, VIn, KOut, VOut> extends RMapReduceExecutor<VIn, KOut, VOut> {
|
||||
|
||||
/**
|
||||
* Setup Mapper object
|
||||
*
|
||||
* @param mapper used during MapReduce
|
||||
* @return self instance
|
||||
*/
|
||||
RMapReduce<KIn, VIn, KOut, VOut> mapper(RMapper<KIn, VIn, KOut, VOut> mapper);
|
||||
|
||||
/**
|
||||
* Setup Reducer object
|
||||
*
|
||||
* @param reducer used during MapReduce
|
||||
* @return self instance
|
||||
*/
|
||||
RMapReduce<KIn, VIn, KOut, VOut> reducer(RReducer<KOut, VOut> reducer);
|
||||
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.redisson.api.RFuture;
|
||||
|
||||
/**
|
||||
* Contains methods for MapReduce process execution.
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <VIn> input value
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public interface RMapReduceExecutor<VIn, KOut, VOut> {
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
*
|
||||
* @return map containing reduced keys and values
|
||||
*/
|
||||
Map<KOut, VOut> execute();
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
* in asynchronous mode
|
||||
*
|
||||
* @return map containing reduced keys and values
|
||||
*/
|
||||
RFuture<Map<KOut, VOut>> executeAsync();
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
* and stores result in map with <code>resultMapName</code>
|
||||
*
|
||||
* @param resultMapName - destination map name
|
||||
*/
|
||||
void execute(String resultMapName);
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
* in asynchronous mode and stores result in map with <code>resultMapName</code>
|
||||
*
|
||||
* @param resultMapName - destination map name
|
||||
* @return void
|
||||
*/
|
||||
RFuture<Void> executeAsync(String resultMapName);
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
* and collides result using defined <code>collator</code>
|
||||
*
|
||||
* @param <R> result type
|
||||
* @param collator applied to result
|
||||
* @return collated result
|
||||
*/
|
||||
<R> R execute(RCollator<KOut, VOut, R> collator);
|
||||
|
||||
/**
|
||||
* Executes MapReduce process across Redisson Nodes
|
||||
* in asynchronous mode and collides result using defined <code>collator</code>
|
||||
*
|
||||
* @param <R> result type
|
||||
* @param collator applied to result
|
||||
* @return collated result
|
||||
*/
|
||||
<R> RFuture<R> executeAsync(RCollator<KOut, VOut, R> collator);
|
||||
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* Mapper task invoked during map phase of MapReduce process and launched across Redisson Nodes.
|
||||
* Every task stores transformed result of input key/value into {@link RCollector} instance.
|
||||
* Collected results are handled by {@link RReducer} instance once
|
||||
* all Mapper tasks have finished.
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KIn> input key
|
||||
* @param <VIn> input value
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public interface RMapper<KIn, VIn, KOut, VOut> extends Serializable {
|
||||
|
||||
/**
|
||||
* Invoked for each Map source entry
|
||||
*
|
||||
* @param key - input key
|
||||
* @param value - input value
|
||||
* @param collector - instance shared across all Mapper tasks
|
||||
*/
|
||||
void map(KIn key, VIn value, RCollector<KOut, VOut> collector);
|
||||
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.api.mapreduce;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* Reduces values mapped by key into single value.
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <K> key type
|
||||
* @param <V> value type
|
||||
*/
|
||||
public interface RReducer<K, V> extends Serializable {
|
||||
|
||||
/**
|
||||
* Invoked for each key
|
||||
*
|
||||
* @param reducedKey - key
|
||||
* @param iter - collection of values
|
||||
* @return value
|
||||
*/
|
||||
V reduce(K reducedKey, Iterator<V> iter);
|
||||
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.redisson.api.RExecutorService;
|
||||
import org.redisson.api.RScheduledExecutorService;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.client.codec.Codec;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KOut> output key
|
||||
* @param <VOut> output value
|
||||
*/
|
||||
public abstract class BaseMapperTask<KOut, VOut> implements Callable<Integer>, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 7559371478909848610L;
|
||||
|
||||
@RInject
|
||||
protected RedissonClient redisson;
|
||||
|
||||
private RReducer<KOut, VOut> reducer;
|
||||
protected String objectName;
|
||||
protected Class<?> objectClass;
|
||||
private Class<?> objectCodecClass;
|
||||
private String semaphoreName;
|
||||
private String resultMapName;
|
||||
|
||||
protected Codec codec;
|
||||
|
||||
public BaseMapperTask() {
|
||||
}
|
||||
|
||||
public BaseMapperTask(RReducer<KOut, VOut> reducer,
|
||||
String mapName, String semaphoreName, String resultMapName, Class<?> mapCodecClass, Class<?> objectClass) {
|
||||
super();
|
||||
this.reducer = reducer;
|
||||
this.objectName = mapName;
|
||||
this.objectCodecClass = mapCodecClass;
|
||||
this.objectClass = objectClass;
|
||||
this.semaphoreName = semaphoreName;
|
||||
this.resultMapName = resultMapName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer call() {
|
||||
try {
|
||||
this.codec = (Codec) objectCodecClass.getConstructor().newInstance();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
|
||||
RScheduledExecutorService executor = redisson.getExecutorService(RExecutorService.MAPREDUCE_NAME);
|
||||
int workersAmount = executor.countActiveWorkers();
|
||||
|
||||
UUID id = UUID.randomUUID();
|
||||
|
||||
RCollector<KOut, VOut> collector = new Collector<KOut, VOut>(codec, redisson, objectName + ":collector:" + id, workersAmount);
|
||||
|
||||
map(collector);
|
||||
|
||||
for (int i = 0; i < workersAmount; i++) {
|
||||
String name = objectName + ":collector:" + id + ":" + i;
|
||||
Runnable runnable = new ReducerTask<KOut, VOut>(name, reducer, objectCodecClass, semaphoreName, resultMapName);
|
||||
executor.submit(runnable);
|
||||
}
|
||||
|
||||
return workersAmount;
|
||||
}
|
||||
|
||||
protected abstract void map(RCollector<KOut, VOut> collector);
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.redisson.api.RMap;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
import org.redisson.api.mapreduce.RCollator;
|
||||
import org.redisson.client.codec.Codec;
|
||||
import org.redisson.misc.Injector;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KOut> key type
|
||||
* @param <VOut> value type
|
||||
* @param <R> result type
|
||||
*/
|
||||
public class CollatorTask<KOut, VOut, R> implements Callable<R> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
private RCollator<KOut, VOut, R> collator;
|
||||
|
||||
private String resultMapName;
|
||||
private Class<?> codecClass;
|
||||
|
||||
private Codec codec;
|
||||
|
||||
public CollatorTask() {
|
||||
}
|
||||
|
||||
public CollatorTask(RCollator<KOut, VOut, R> collator, String resultMapName, Class<?> codecClass) {
|
||||
super();
|
||||
this.collator = collator;
|
||||
this.resultMapName = resultMapName;
|
||||
this.codecClass = codecClass;
|
||||
}
|
||||
|
||||
@Override
|
||||
public R call() throws Exception {
|
||||
try {
|
||||
this.codec = (Codec) codecClass.getConstructor().newInstance();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
|
||||
Injector.inject(collator, redisson);
|
||||
|
||||
RMap<KOut, VOut> resultMap = redisson.getMap(resultMapName, codec);
|
||||
R result = collator.collate(resultMap);
|
||||
resultMap.delete();
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import org.redisson.api.RLexSortedSet;
|
||||
import org.redisson.api.RList;
|
||||
import org.redisson.api.RScoredSortedSet;
|
||||
import org.redisson.api.RSet;
|
||||
import org.redisson.api.RSetCache;
|
||||
import org.redisson.api.RSortedSet;
|
||||
import org.redisson.api.mapreduce.RCollectionMapper;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.misc.Injector;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <VIn> input value type
|
||||
* @param <KOut> output key type
|
||||
* @param <VOut> output value type
|
||||
*/
|
||||
public class CollectionMapperTask<VIn, KOut, VOut> extends BaseMapperTask<KOut, VOut> {
|
||||
|
||||
private static final long serialVersionUID = -2634049426877164580L;
|
||||
|
||||
RCollectionMapper<VIn, KOut, VOut> mapper;
|
||||
|
||||
public CollectionMapperTask() {
|
||||
}
|
||||
|
||||
public CollectionMapperTask(RCollectionMapper<VIn, KOut, VOut> mapper, RReducer<KOut, VOut> reducer, String mapName, String semaphoreName, String resultMapName,
|
||||
Class<?> mapCodecClass, Class<?> mapClass) {
|
||||
super(reducer, mapName, semaphoreName, resultMapName, mapCodecClass, mapClass);
|
||||
this.mapper = mapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void map(RCollector<KOut, VOut> collector) {
|
||||
Injector.inject(mapper, redisson);
|
||||
|
||||
Iterable<VIn> collection = null;
|
||||
if (RSetCache.class.isAssignableFrom(objectClass)) {
|
||||
collection = redisson.getSetCache(objectName, codec);
|
||||
} else if (RSet.class.isAssignableFrom(objectClass)) {
|
||||
collection = redisson.getSet(objectName, codec);
|
||||
} else if (RSortedSet.class.isAssignableFrom(objectClass)) {
|
||||
collection = redisson.getSortedSet(objectName, codec);
|
||||
} else if (RScoredSortedSet.class.isAssignableFrom(objectClass)) {
|
||||
collection = redisson.getScoredSortedSet(objectName, codec);
|
||||
} else if (RLexSortedSet.class.isAssignableFrom(objectClass)) {
|
||||
collection = (Iterable<VIn>) redisson.getLexSortedSet(objectName);
|
||||
} else if (RList.class.isAssignableFrom(objectClass)) {
|
||||
collection = redisson.getList(objectName, codec);
|
||||
} else {
|
||||
throw new IllegalStateException("Unable to work with " + objectClass);
|
||||
}
|
||||
|
||||
for (VIn value : collection) {
|
||||
mapper.map(value, collector);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.redisson.api.RListMultimap;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.client.codec.Codec;
|
||||
|
||||
import net.openhft.hashing.LongHashFunction;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <K> key
|
||||
* @param <V> value
|
||||
*/
|
||||
public class Collector<K, V> implements RCollector<K, V> {
|
||||
|
||||
private RedissonClient client;
|
||||
private String name;
|
||||
private int parts;
|
||||
private Codec codec;
|
||||
|
||||
public Collector(Codec codec, RedissonClient client, String name, int parts) {
|
||||
super();
|
||||
this.client = client;
|
||||
this.name = name;
|
||||
this.parts = parts;
|
||||
this.codec = codec;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void emit(K key, V value) {
|
||||
try {
|
||||
byte[] encodedKey = codec.getValueEncoder().encode(key);
|
||||
long hash = LongHashFunction.xx_r39().hashBytes(encodedKey);
|
||||
String partName = name + ":" + Math.abs(hash % parts);
|
||||
|
||||
RListMultimap<K, V> multimap = client.getListMultimap(partName, codec);
|
||||
multimap.put(key, value);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.redisson.api.RBatch;
|
||||
import org.redisson.api.RExecutorService;
|
||||
import org.redisson.api.RFuture;
|
||||
import org.redisson.api.RMapAsync;
|
||||
import org.redisson.api.RObject;
|
||||
import org.redisson.api.RSemaphore;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.mapreduce.RCollator;
|
||||
import org.redisson.api.mapreduce.RMapReduceExecutor;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.client.codec.Codec;
|
||||
import org.redisson.connection.ConnectionManager;
|
||||
import org.redisson.misc.RPromise;
|
||||
import org.redisson.misc.TransferListener;
|
||||
|
||||
import io.netty.util.concurrent.Future;
|
||||
import io.netty.util.concurrent.FutureListener;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <M> mapper type
|
||||
* @param <VIn> input value type
|
||||
* @param <KOut> output key type
|
||||
* @param <VOut> output value type
|
||||
*/
|
||||
abstract class MapReduceExecutor<M, VIn, KOut, VOut> implements RMapReduceExecutor<VIn, KOut, VOut> {
|
||||
|
||||
private final RedissonClient redisson;
|
||||
private final RExecutorService executorService;
|
||||
final String semaphoreName;
|
||||
final String resultMapName;
|
||||
|
||||
final Codec objectCodec;
|
||||
final String objectName;
|
||||
final Class<?> objectClass;
|
||||
|
||||
|
||||
private ConnectionManager connectionManager;
|
||||
RReducer<KOut, VOut> reducer;
|
||||
M mapper;
|
||||
|
||||
public MapReduceExecutor(RObject object, RedissonClient redisson, ConnectionManager connectionManager) {
|
||||
this.objectName = object.getName();
|
||||
this.objectCodec = object.getCodec();
|
||||
this.objectClass = object.getClass();
|
||||
|
||||
this.redisson = redisson;
|
||||
UUID id = UUID.randomUUID();
|
||||
this.semaphoreName = object.getName() + ":semaphore:" + id;
|
||||
this.resultMapName = object.getName() + ":result:" + id;
|
||||
this.executorService = redisson.getExecutorService(RExecutorService.MAPREDUCE_NAME);
|
||||
this.connectionManager = connectionManager;
|
||||
}
|
||||
|
||||
protected void check(Object task) {
|
||||
if (task == null) {
|
||||
throw new NullPointerException("Task is not defined");
|
||||
}
|
||||
if (task.getClass().isAnonymousClass()) {
|
||||
throw new IllegalArgumentException("Task can't be created using anonymous class");
|
||||
}
|
||||
if (task.getClass().isMemberClass()
|
||||
&& !Modifier.isStatic(task.getClass().getModifiers())) {
|
||||
throw new IllegalArgumentException("Task class is an inner class and it should be static");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<KOut, VOut> execute() {
|
||||
return connectionManager.getCommandExecutor().get(executeAsync());
|
||||
}
|
||||
|
||||
@Override
|
||||
public RFuture<Map<KOut, VOut>> executeAsync() {
|
||||
final RPromise<Map<KOut, VOut>> promise = connectionManager.newPromise();
|
||||
executeMapperAsync(resultMapName).addListener(new FutureListener<Void>() {
|
||||
@Override
|
||||
public void operationComplete(Future<Void> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
RBatch batch = redisson.createBatch();
|
||||
RMapAsync<KOut, VOut> resultMap = batch.getMap(resultMapName, objectCodec);
|
||||
resultMap.readAllMapAsync().addListener(new TransferListener<Map<KOut, VOut>>(promise));
|
||||
resultMap.deleteAsync();
|
||||
batch.executeAsync();
|
||||
}
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(String resultMapName) {
|
||||
connectionManager.getCommandExecutor().get(executeAsync(resultMapName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public RFuture<Void> executeAsync(String resultMapName) {
|
||||
final RPromise<Void> promise = connectionManager.newPromise();
|
||||
executeMapperAsync(resultMapName).addListener(new FutureListener<Void>() {
|
||||
|
||||
@Override
|
||||
public void operationComplete(Future<Void> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
promise.trySuccess(null);
|
||||
}
|
||||
});
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
|
||||
private RPromise<Void> executeMapperAsync(String resultMapName) {
|
||||
if (mapper == null) {
|
||||
throw new NullPointerException("Mapper is not defined");
|
||||
}
|
||||
if (reducer == null) {
|
||||
throw new NullPointerException("Reducer is not defined");
|
||||
}
|
||||
|
||||
final RPromise<Void> promise = connectionManager.newPromise();
|
||||
Callable<Integer> task = createTask(resultMapName);
|
||||
executorService.submit(task).addListener(new FutureListener<Integer>() {
|
||||
@Override
|
||||
public void operationComplete(Future<Integer> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
Integer workers = future.getNow();
|
||||
final RSemaphore semaphore = redisson.getSemaphore(semaphoreName);
|
||||
semaphore.acquireAsync(workers).addListener(new FutureListener<Void>() {
|
||||
@Override
|
||||
public void operationComplete(Future<Void> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
semaphore.deleteAsync().addListener(new FutureListener<Boolean>() {
|
||||
@Override
|
||||
public void operationComplete(Future<Boolean> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
promise.trySuccess(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
|
||||
protected abstract Callable<Integer> createTask(String resultMapName);
|
||||
|
||||
@Override
|
||||
public <R> R execute(RCollator<KOut, VOut, R> collator) {
|
||||
return connectionManager.getCommandExecutor().get(executeAsync(collator));
|
||||
}
|
||||
|
||||
@Override
|
||||
public <R> RFuture<R> executeAsync(final RCollator<KOut, VOut, R> collator) {
|
||||
check(collator);
|
||||
|
||||
final RPromise<R> promise = connectionManager.newPromise();
|
||||
executeMapperAsync(resultMapName).addListener(new FutureListener<Void>() {
|
||||
@Override
|
||||
public void operationComplete(Future<Void> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
Callable<R> collatorTask = new CollatorTask<KOut, VOut, R>(collator, resultMapName, objectCodec.getClass());
|
||||
executorService.submit(collatorTask).addListener(new FutureListener<R>() {
|
||||
@Override
|
||||
public void operationComplete(Future<R> future) throws Exception {
|
||||
if (!future.isSuccess()) {
|
||||
promise.tryFailure(future.cause());
|
||||
return;
|
||||
}
|
||||
|
||||
promise.trySuccess(future.getNow());
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.redisson.api.RMap;
|
||||
import org.redisson.api.RMapCache;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.api.mapreduce.RMapper;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.misc.Injector;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KIn> input key type
|
||||
* @param <VIn> input key type
|
||||
* @param <KOut> output key type
|
||||
* @param <VOut> output key type
|
||||
*/
|
||||
public class MapperTask<KIn, VIn, KOut, VOut> extends BaseMapperTask<KOut, VOut> {
|
||||
|
||||
private static final long serialVersionUID = 2441161019495880394L;
|
||||
|
||||
RMapper<KIn, VIn, KOut, VOut> mapper;
|
||||
|
||||
public MapperTask() {
|
||||
}
|
||||
|
||||
public MapperTask(RMapper<KIn, VIn, KOut, VOut> mapper, RReducer<KOut, VOut> reducer, String mapName, String semaphoreName, String resultMapName,
|
||||
Class<?> mapCodecClass, Class<?> mapClass) {
|
||||
super(reducer, mapName, semaphoreName, resultMapName, mapCodecClass, mapClass);
|
||||
this.mapper = mapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void map(RCollector<KOut, VOut> collector) {
|
||||
Injector.inject(mapper, redisson);
|
||||
|
||||
RMap<KIn, VIn> map = null;
|
||||
if (RMapCache.class.isAssignableFrom(objectClass)) {
|
||||
map = redisson.getMapCache(objectName, codec);
|
||||
} else {
|
||||
map = redisson.getMap(objectName, codec);
|
||||
}
|
||||
|
||||
for (Entry<KIn, VIn> entry : map.entrySet()) {
|
||||
mapper.map(entry.getKey(), entry.getValue(), collector);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import org.redisson.api.RObject;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.mapreduce.RCollectionMapReduce;
|
||||
import org.redisson.api.mapreduce.RCollectionMapper;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.connection.ConnectionManager;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <VIn> input value type
|
||||
* @param <KOut> output key type
|
||||
* @param <VOut> output value type
|
||||
*/
|
||||
public class RedissonCollectionMapReduce<VIn, KOut, VOut> extends MapReduceExecutor<RCollectionMapper<VIn, KOut, VOut>, VIn, KOut, VOut>
|
||||
implements RCollectionMapReduce<VIn, KOut, VOut> {
|
||||
|
||||
public RedissonCollectionMapReduce(RObject object, RedissonClient redisson, ConnectionManager connectionManager) {
|
||||
super(object, redisson, connectionManager);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RCollectionMapReduce<VIn, KOut, VOut> mapper(RCollectionMapper<VIn, KOut, VOut> mapper) {
|
||||
check(mapper);
|
||||
this.mapper = mapper;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RCollectionMapReduce<VIn, KOut, VOut> reducer(RReducer<KOut, VOut> reducer) {
|
||||
check(reducer);
|
||||
this.reducer = reducer;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectionMapperTask<VIn, KOut, VOut> createTask(String resultMapName) {
|
||||
return new CollectionMapperTask<VIn, KOut, VOut>(mapper, reducer, objectName, semaphoreName, resultMapName, objectCodec.getClass(), objectClass);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import org.redisson.api.RFuture;
|
||||
import org.redisson.api.RObject;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.mapreduce.RMapReduce;
|
||||
import org.redisson.api.mapreduce.RMapper;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.connection.ConnectionManager;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KIn> input key type
|
||||
* @param <VIn> input value type
|
||||
* @param <KOut> output key type
|
||||
* @param <VOut> output value type
|
||||
*/
|
||||
public class RedissonMapReduce<KIn, VIn, KOut, VOut> extends MapReduceExecutor<RMapper<KIn, VIn, KOut, VOut>, VIn, KOut, VOut>
|
||||
implements RMapReduce<KIn, VIn, KOut, VOut> {
|
||||
|
||||
public RedissonMapReduce(RObject object, RedissonClient redisson, ConnectionManager connectionManager) {
|
||||
super(object, redisson, connectionManager);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RMapReduce<KIn, VIn, KOut, VOut> mapper(RMapper<KIn, VIn, KOut, VOut> mapper) {
|
||||
check(mapper);
|
||||
this.mapper = mapper;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RMapReduce<KIn, VIn, KOut, VOut> reducer(RReducer<KOut, VOut> reducer) {
|
||||
check(reducer);
|
||||
this.reducer = reducer;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Callable<Integer> createTask(String resultMapName) {
|
||||
return new MapperTask<KIn, VIn, KOut, VOut>(mapper, reducer, objectName, semaphoreName, resultMapName, objectCodec.getClass(), objectClass);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.mapreduce;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
import org.redisson.api.RListMultimap;
|
||||
import org.redisson.api.RMap;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
import org.redisson.client.codec.Codec;
|
||||
import org.redisson.misc.Injector;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
* @param <KOut> key
|
||||
* @param <VOut> value
|
||||
*/
|
||||
public class ReducerTask<KOut, VOut> implements Runnable, Serializable {
|
||||
|
||||
private static final long serialVersionUID = 3556632668150314703L;
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
private String name;
|
||||
private String semaphoreName;
|
||||
private String resultMapName;
|
||||
private RReducer<KOut, VOut> reducer;
|
||||
private Class<?> codecClass;
|
||||
private Codec codec;
|
||||
|
||||
public ReducerTask() {
|
||||
}
|
||||
|
||||
public ReducerTask(String name, RReducer<KOut, VOut> reducer, Class<?> codecClass, String semaphoreName, String resultMapName) {
|
||||
this.name = name;
|
||||
this.reducer = reducer;
|
||||
this.semaphoreName = semaphoreName;
|
||||
this.resultMapName = resultMapName;
|
||||
this.codecClass = codecClass;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
this.codec = (Codec) codecClass.getConstructor().newInstance();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
|
||||
Injector.inject(reducer, redisson);
|
||||
|
||||
RMap<KOut, VOut> map = redisson.getMap(resultMapName);
|
||||
RListMultimap<KOut, VOut> multimap = redisson.getListMultimap(name, codec);
|
||||
for (KOut key : multimap.keySet()) {
|
||||
List<VOut> values = multimap.get(key);
|
||||
VOut out = reducer.reduce(key, values.iterator());
|
||||
map.put(key, out);
|
||||
}
|
||||
multimap.delete();
|
||||
redisson.getSemaphore(semaphoreName).release();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Copyright 2016 Nikita Koksharov
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.redisson.misc;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Nikita Koksharov
|
||||
*
|
||||
*/
|
||||
public class Injector {
|
||||
|
||||
public static void inject(Object task, RedissonClient redisson) {
|
||||
List<Field> allFields = new ArrayList<Field>();
|
||||
Class<?> clazz = task.getClass();
|
||||
while (true) {
|
||||
if (clazz != null) {
|
||||
Field[] fields = clazz.getDeclaredFields();
|
||||
allFields.addAll(Arrays.asList(fields));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
if (clazz.getSuperclass() != Object.class) {
|
||||
clazz = clazz.getSuperclass();
|
||||
} else {
|
||||
clazz = null;
|
||||
}
|
||||
}
|
||||
|
||||
for (Field field : allFields) {
|
||||
if (RedissonClient.class.isAssignableFrom(field.getType())
|
||||
&& field.isAnnotationPresent(RInject.class)) {
|
||||
field.setAccessible(true);
|
||||
try {
|
||||
field.set(task, redisson);
|
||||
} catch (IllegalAccessException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,229 @@
|
||||
package org.redisson;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.redisson.api.RExecutorService;
|
||||
import org.redisson.api.RList;
|
||||
import org.redisson.api.RMap;
|
||||
import org.redisson.api.RQueue;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
import org.redisson.api.mapreduce.RCollator;
|
||||
import org.redisson.api.mapreduce.RCollectionMapReduce;
|
||||
import org.redisson.api.mapreduce.RCollectionMapper;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class RedissonCollectionMapReduceTest extends BaseTest {
|
||||
|
||||
public static class WordMapper implements RCollectionMapper<String, String, Integer> {
|
||||
|
||||
@Override
|
||||
public void map(String value, RCollector<String, Integer> collector) {
|
||||
String[] words = value.split("[^a-zA-Z]");
|
||||
for (String word : words) {
|
||||
collector.emit(word, 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordReducer implements RReducer<String, Integer> {
|
||||
|
||||
@Override
|
||||
public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
int sum = 0;
|
||||
while (iter.hasNext()) {
|
||||
Integer i = (Integer) iter.next();
|
||||
sum += i;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordCollator implements RCollator<String, Integer, Integer> {
|
||||
|
||||
@Override
|
||||
public Integer collate(Map<String, Integer> resultMap) {
|
||||
int result = 0;
|
||||
for (Integer count : resultMap.values()) {
|
||||
result += count;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "{index} - {0}")
|
||||
public static Iterable<Object[]> mapClasses() {
|
||||
return Arrays.asList(new Object[][]{
|
||||
{RList.class}, {RQueue.class}
|
||||
});
|
||||
}
|
||||
|
||||
@Parameterized.Parameter(0)
|
||||
public Class<?> mapClass;
|
||||
|
||||
|
||||
@Before
|
||||
public void beforeTest() {
|
||||
redisson.getExecutorService(RExecutorService.MAPREDUCE_NAME).registerWorkers(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
RList<String> list = getCollection();
|
||||
|
||||
list.add("Alice was beginning to get very tired");
|
||||
list.add("of sitting by her sister on the bank and");
|
||||
list.add("of having nothing to do once or twice she");
|
||||
list.add("had peeped into the book her sister was reading");
|
||||
list.add("but it had no pictures or conversations in it");
|
||||
list.add("and what is the use of a book");
|
||||
list.add("thought Alice without pictures or conversation");
|
||||
|
||||
Map<String, Integer> result = new HashMap<>();
|
||||
result.put("to", 2);
|
||||
result.put("Alice", 2);
|
||||
result.put("get", 1);
|
||||
result.put("beginning", 1);
|
||||
result.put("sitting", 1);
|
||||
result.put("do", 1);
|
||||
result.put("by", 1);
|
||||
result.put("or", 3);
|
||||
result.put("into", 1);
|
||||
result.put("sister", 2);
|
||||
result.put("on", 1);
|
||||
result.put("a", 1);
|
||||
result.put("without", 1);
|
||||
result.put("and", 2);
|
||||
result.put("once", 1);
|
||||
result.put("twice", 1);
|
||||
result.put("she", 1);
|
||||
result.put("had", 2);
|
||||
result.put("reading", 1);
|
||||
result.put("but", 1);
|
||||
result.put("it", 2);
|
||||
result.put("no", 1);
|
||||
result.put("in", 1);
|
||||
result.put("what", 1);
|
||||
result.put("use", 1);
|
||||
result.put("thought", 1);
|
||||
result.put("conversation", 1);
|
||||
result.put("was", 2);
|
||||
result.put("very", 1);
|
||||
result.put("tired", 1);
|
||||
result.put("of", 3);
|
||||
result.put("her", 2);
|
||||
result.put("the", 3);
|
||||
result.put("bank", 1);
|
||||
result.put("having", 1);
|
||||
result.put("nothing", 1);
|
||||
result.put("peeped", 1);
|
||||
result.put("book", 2);
|
||||
result.put("pictures", 2);
|
||||
result.put("conversations", 1);
|
||||
result.put("is", 1);
|
||||
|
||||
RCollectionMapReduce<String, String, Integer> mapReduce = list.<String, Integer>mapReduce().mapper(new WordMapper()).reducer(new WordReducer());
|
||||
assertThat(mapReduce.execute()).isEqualTo(result);
|
||||
Integer count = mapReduce.execute(new WordCollator());
|
||||
assertThat(count).isEqualTo(57);
|
||||
|
||||
mapReduce.execute("resultMap");
|
||||
RMap<Object, Object> resultMap = redisson.getMap("resultMap");
|
||||
assertThat(resultMap).isEqualTo(result);
|
||||
resultMap.delete();
|
||||
}
|
||||
|
||||
public static class WordMapperInject implements RCollectionMapper<String, String, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public void map(String value, RCollector<String, Integer> collector) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
String[] words = value.split("[^a-zA-Z]");
|
||||
for (String word : words) {
|
||||
collector.emit(word, 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordReducerInject implements RReducer<String, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
int sum = 0;
|
||||
while (iter.hasNext()) {
|
||||
Integer i = (Integer) iter.next();
|
||||
sum += i;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordCollatorInject implements RCollator<String, Integer, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public Integer collate(Map<String, Integer> resultMap) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
int result = 0;
|
||||
for (Integer count : resultMap.values()) {
|
||||
result += count;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInjector() {
|
||||
RList<String> list = getCollection();
|
||||
|
||||
list.add("Alice was beginning to get very tired");
|
||||
|
||||
RCollectionMapReduce<String, String, Integer> mapReduce = list.<String, Integer>mapReduce().mapper(new WordMapperInject()).reducer(new WordReducerInject());
|
||||
|
||||
mapReduce.execute();
|
||||
assertThat(redisson.getAtomicLong("test").get()).isEqualTo(8);
|
||||
|
||||
mapReduce.execute(new WordCollatorInject());
|
||||
assertThat(redisson.getAtomicLong("test").get()).isEqualTo(16 + 1);
|
||||
}
|
||||
|
||||
private RList<String> getCollection() {
|
||||
RList<String> list = null;
|
||||
if (RList.class.isAssignableFrom(mapClass)) {
|
||||
list = redisson.getList("list");
|
||||
} else if (RQueue.class.isAssignableFrom(mapClass)) {
|
||||
list = (RList<String>) redisson.<String>getQueue("queue");
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,228 @@
|
||||
package org.redisson;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.redisson.api.RExecutorService;
|
||||
import org.redisson.api.RMap;
|
||||
import org.redisson.api.RMapCache;
|
||||
import org.redisson.api.RedissonClient;
|
||||
import org.redisson.api.annotation.RInject;
|
||||
import org.redisson.api.mapreduce.RCollator;
|
||||
import org.redisson.api.mapreduce.RCollector;
|
||||
import org.redisson.api.mapreduce.RMapReduce;
|
||||
import org.redisson.api.mapreduce.RMapper;
|
||||
import org.redisson.api.mapreduce.RReducer;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class RedissonMapReduceTest extends BaseTest {
|
||||
|
||||
public static class WordMapper implements RMapper<String, String, String, Integer> {
|
||||
|
||||
@Override
|
||||
public void map(String key, String value, RCollector<String, Integer> collector) {
|
||||
String[] words = value.split("[^a-zA-Z]");
|
||||
for (String word : words) {
|
||||
collector.emit(word, 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordReducer implements RReducer<String, Integer> {
|
||||
|
||||
@Override
|
||||
public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
int sum = 0;
|
||||
while (iter.hasNext()) {
|
||||
Integer i = (Integer) iter.next();
|
||||
sum += i;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordCollator implements RCollator<String, Integer, Integer> {
|
||||
|
||||
@Override
|
||||
public Integer collate(Map<String, Integer> resultMap) {
|
||||
int result = 0;
|
||||
for (Integer count : resultMap.values()) {
|
||||
result += count;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "{index} - {0}")
|
||||
public static Iterable<Object[]> mapClasses() {
|
||||
return Arrays.asList(new Object[][]{
|
||||
{RMap.class}, {RMapCache.class}
|
||||
});
|
||||
}
|
||||
|
||||
@Parameterized.Parameter(0)
|
||||
public Class<?> mapClass;
|
||||
|
||||
|
||||
@Before
|
||||
public void beforeTest() {
|
||||
redisson.getExecutorService(RExecutorService.MAPREDUCE_NAME).registerWorkers(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
RMap<String, String> map = null;
|
||||
if (RMapCache.class.isAssignableFrom(mapClass)) {
|
||||
map = redisson.getMapCache("map");
|
||||
} else {
|
||||
map = redisson.getMap("map");
|
||||
}
|
||||
|
||||
map.put("1", "Alice was beginning to get very tired");
|
||||
map.put("2", "of sitting by her sister on the bank and");
|
||||
map.put("3", "of having nothing to do once or twice she");
|
||||
map.put("4", "had peeped into the book her sister was reading");
|
||||
map.put("5", "but it had no pictures or conversations in it");
|
||||
map.put("6", "and what is the use of a book");
|
||||
map.put("7", "thought Alice without pictures or conversation");
|
||||
|
||||
Map<String, Integer> result = new HashMap<>();
|
||||
result.put("to", 2);
|
||||
result.put("Alice", 2);
|
||||
result.put("get", 1);
|
||||
result.put("beginning", 1);
|
||||
result.put("sitting", 1);
|
||||
result.put("do", 1);
|
||||
result.put("by", 1);
|
||||
result.put("or", 3);
|
||||
result.put("into", 1);
|
||||
result.put("sister", 2);
|
||||
result.put("on", 1);
|
||||
result.put("a", 1);
|
||||
result.put("without", 1);
|
||||
result.put("and", 2);
|
||||
result.put("once", 1);
|
||||
result.put("twice", 1);
|
||||
result.put("she", 1);
|
||||
result.put("had", 2);
|
||||
result.put("reading", 1);
|
||||
result.put("but", 1);
|
||||
result.put("it", 2);
|
||||
result.put("no", 1);
|
||||
result.put("in", 1);
|
||||
result.put("what", 1);
|
||||
result.put("use", 1);
|
||||
result.put("thought", 1);
|
||||
result.put("conversation", 1);
|
||||
result.put("was", 2);
|
||||
result.put("very", 1);
|
||||
result.put("tired", 1);
|
||||
result.put("of", 3);
|
||||
result.put("her", 2);
|
||||
result.put("the", 3);
|
||||
result.put("bank", 1);
|
||||
result.put("having", 1);
|
||||
result.put("nothing", 1);
|
||||
result.put("peeped", 1);
|
||||
result.put("book", 2);
|
||||
result.put("pictures", 2);
|
||||
result.put("conversations", 1);
|
||||
result.put("is", 1);
|
||||
|
||||
RMapReduce<String, String, String, Integer> mapReduce = map.<String, Integer>mapReduce().mapper(new WordMapper()).reducer(new WordReducer());
|
||||
assertThat(mapReduce.execute()).isEqualTo(result);
|
||||
Integer count = mapReduce.execute(new WordCollator());
|
||||
assertThat(count).isEqualTo(57);
|
||||
|
||||
mapReduce.execute("resultMap");
|
||||
RMap<Object, Object> resultMap = redisson.getMap("resultMap");
|
||||
assertThat(resultMap).isEqualTo(result);
|
||||
resultMap.delete();
|
||||
}
|
||||
|
||||
public static class WordMapperInject implements RMapper<String, String, String, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public void map(String key, String value, RCollector<String, Integer> collector) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
String[] words = value.split("[^a-zA-Z]");
|
||||
for (String word : words) {
|
||||
collector.emit(word, 1);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordReducerInject implements RReducer<String, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public Integer reduce(String reducedKey, Iterator<Integer> iter) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
int sum = 0;
|
||||
while (iter.hasNext()) {
|
||||
Integer i = (Integer) iter.next();
|
||||
sum += i;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class WordCollatorInject implements RCollator<String, Integer, Integer> {
|
||||
|
||||
@RInject
|
||||
private RedissonClient redisson;
|
||||
|
||||
@Override
|
||||
public Integer collate(Map<String, Integer> resultMap) {
|
||||
redisson.getAtomicLong("test").incrementAndGet();
|
||||
|
||||
int result = 0;
|
||||
for (Integer count : resultMap.values()) {
|
||||
result += count;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInjector() {
|
||||
RMap<String, String> map = null;
|
||||
if (RMapCache.class.isAssignableFrom(mapClass)) {
|
||||
map = redisson.getMapCache("map");
|
||||
} else {
|
||||
map = redisson.getMap("map");
|
||||
}
|
||||
|
||||
map.put("1", "Alice was beginning to get very tired");
|
||||
|
||||
RMapReduce<String, String, String, Integer> mapReduce = map.<String, Integer>mapReduce().mapper(new WordMapperInject()).reducer(new WordReducerInject());
|
||||
|
||||
mapReduce.execute();
|
||||
assertThat(redisson.getAtomicLong("test").get()).isEqualTo(8);
|
||||
|
||||
mapReduce.execute(new WordCollatorInject());
|
||||
assertThat(redisson.getAtomicLong("test").get()).isEqualTo(16 + 1);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue