/**
* Create superstep classes by initiazling from current state in configuration
*
* @param conf Configuration
* @return Superstep classes
*/
public static SuperstepClasses createAndExtractTypes(ImmutableClassesGiraphConfiguration conf) {
return new SuperstepClasses(
conf,
conf.getComputationClass(),
conf.getOutgoingMessageClasses(),
conf.getOutgoingMessageClasses().createCopyForNewSuperstep());
}
/**
* Verify that types of current Computation and MessageCombiner are valid. If types don't match an
* {@link IllegalStateException} will be thrown.
*
* @param checkMatchingMesssageTypes Check that the incoming/outgoing message types match
*/
public void verifyTypesMatch(boolean checkMatchingMesssageTypes) {
// In some cases, for example when using Jython, the Computation class may
// not be set. This is because it is created by a ComputationFactory
// dynamically and not known ahead of time. In this case there is nothing to
// verify here so we bail.
if (COMPUTATION_LANGUAGE.get(conf) == Language.JYTHON) {
return;
}
Class<?>[] computationTypes =
ReflectionUtils.getTypeArguments(TypesHolder.class, computationClass);
ReflectionUtils.verifyTypes(
conf.getVertexIdClass(), computationTypes[0], "Vertex id", computationClass);
ReflectionUtils.verifyTypes(
conf.getVertexValueClass(), computationTypes[1], "Vertex value", computationClass);
ReflectionUtils.verifyTypes(
conf.getEdgeValueClass(), computationTypes[2], "Edge value", computationClass);
if (checkMatchingMesssageTypes) {
ReflectionUtils.verifyTypes(
incomingMessageClasses.getMessageClass(),
computationTypes[3],
"Incoming message type",
computationClass);
}
ReflectionUtils.verifyTypes(
outgoingMessageClasses.getMessageClass(),
computationTypes[4],
"Outgoing message type",
computationClass);
outgoingMessageClasses.verifyConsistent(conf);
}
/**
* Constructor.
*
* @param service Service worker
* @param configuration Configuration
* @param progressable Progressable
*/
public AbstractEdgeStore(
CentralizedServiceWorker<I, V, E> service,
ImmutableClassesGiraphConfiguration<I, V, E> configuration,
Progressable progressable) {
this.service = service;
this.configuration = configuration;
this.progressable = progressable;
transientEdges =
new MapMaker()
.concurrencyLevel(configuration.getNettyServerExecutionConcurrency())
.makeMap();
reuseEdgeObjects = configuration.reuseEdgeObjects();
useInputOutEdges = configuration.useInputOutEdges();
}
/**
* Convert the input edges to the {@link OutEdges} data structure used for computation (if
* different).
*
* @param inputEdges Input edges
* @return Compute edges
*/
private OutEdges<I, E> convertInputToComputeEdges(OutEdges<I, E> inputEdges) {
if (!useInputOutEdges) {
return inputEdges;
} else {
return configuration.createAndInitializeOutEdges(inputEdges);
}
}
@Override
public void moveEdgesToVertices() {
final boolean createSourceVertex = configuration.getCreateSourceVertex();
if (transientEdges.isEmpty()) {
if (LOG.isInfoEnabled()) {
LOG.info("moveEdgesToVertices: No edges to move");
}
return;
}
if (LOG.isInfoEnabled()) {
LOG.info("moveEdgesToVertices: Moving incoming edges to vertices.");
}
final BlockingQueue<Integer> partitionIdQueue = new ArrayBlockingQueue<>(transientEdges.size());
partitionIdQueue.addAll(transientEdges.keySet());
int numThreads = configuration.getNumInputSplitsThreads();
CallableFactory<Void> callableFactory =
new CallableFactory<Void>() {
@Override
public Callable<Void> newCallable(int callableId) {
return new Callable<Void>() {
@Override
public Void call() throws Exception {
Integer partitionId;
I representativeVertexId = configuration.createVertexId();
while ((partitionId = partitionIdQueue.poll()) != null) {
Partition<I, V, E> partition =
service.getPartitionStore().getOrCreatePartition(partitionId);
Map<K, OutEdges<I, E>> partitionEdges = transientEdges.remove(partitionId);
Iterator<Et> iterator = getPartitionEdgesIterator(partitionEdges);
// process all vertices in given partition
while (iterator.hasNext()) {
Et entry = iterator.next();
I vertexId = getVertexId(entry, representativeVertexId);
OutEdges<I, E> outEdges =
convertInputToComputeEdges(removePartitionEdges(entry, partitionEdges));
Vertex<I, V, E> vertex = partition.getVertex(vertexId);
// If the source vertex doesn't exist, create it. Otherwise,
// just set the edges.
if (vertex == null) {
if (createSourceVertex) {
// createVertex only if it is allowed by configuration
vertex = configuration.createVertex();
vertex.initialize(
createVertexId(entry), configuration.createVertexValue(), outEdges);
partition.putVertex(vertex);
}
} else {
// A vertex may exist with or without edges initially
// and optimize the case of no initial edges
if (vertex.getNumEdges() == 0) {
vertex.setEdges(outEdges);
} else {
for (Edge<I, E> edge : outEdges) {
vertex.addEdge(edge);
}
}
// Some Partition implementations (e.g. ByteArrayPartition)
// require us to put back the vertex after modifying it.
partition.saveVertex(vertex);
}
}
// Some PartitionStore implementations
// (e.g. DiskBackedPartitionStore) require us to put back the
// partition after modifying it.
service.getPartitionStore().putPartition(partition);
}
return null;
}
};
}
};
ProgressableUtils.getResultsWithNCallables(
callableFactory, numThreads, "move-edges-%d", progressable);
// remove all entries
transientEdges.clear();
if (LOG.isInfoEnabled()) {
LOG.info("moveEdgesToVertices: Finished moving incoming edges to " + "vertices.");
}
}
/**
* Factory method to create a new wrapper over the existing out-edges.
*
* @param edges Current out-edges
* @param conf Configuration
* @param <I> Vertex id
* @param <E> Edge value
* @return The wrapped edges
*/
public static <I extends WritableComparable, E extends Writable> MutableEdgesWrapper<I, E> wrap(
OutEdges<I, E> edges, ImmutableClassesGiraphConfiguration<I, ?, E, ?> conf) {
MutableEdgesWrapper<I, E> wrapper =
new MutableEdgesWrapper<I, E>(edges, conf.createAndInitializeOutEdges(edges.size()));
return wrapper;
}