/** * Create superstep classes by initiazling from current state in configuration * * @param conf Configuration * @return Superstep classes */ public static SuperstepClasses createAndExtractTypes(ImmutableClassesGiraphConfiguration conf) { return new SuperstepClasses( conf, conf.getComputationClass(), conf.getOutgoingMessageClasses(), conf.getOutgoingMessageClasses().createCopyForNewSuperstep()); }
/** * Verify that types of current Computation and MessageCombiner are valid. If types don't match an * {@link IllegalStateException} will be thrown. * * @param checkMatchingMesssageTypes Check that the incoming/outgoing message types match */ public void verifyTypesMatch(boolean checkMatchingMesssageTypes) { // In some cases, for example when using Jython, the Computation class may // not be set. This is because it is created by a ComputationFactory // dynamically and not known ahead of time. In this case there is nothing to // verify here so we bail. if (COMPUTATION_LANGUAGE.get(conf) == Language.JYTHON) { return; } Class<?>[] computationTypes = ReflectionUtils.getTypeArguments(TypesHolder.class, computationClass); ReflectionUtils.verifyTypes( conf.getVertexIdClass(), computationTypes[0], "Vertex id", computationClass); ReflectionUtils.verifyTypes( conf.getVertexValueClass(), computationTypes[1], "Vertex value", computationClass); ReflectionUtils.verifyTypes( conf.getEdgeValueClass(), computationTypes[2], "Edge value", computationClass); if (checkMatchingMesssageTypes) { ReflectionUtils.verifyTypes( incomingMessageClasses.getMessageClass(), computationTypes[3], "Incoming message type", computationClass); } ReflectionUtils.verifyTypes( outgoingMessageClasses.getMessageClass(), computationTypes[4], "Outgoing message type", computationClass); outgoingMessageClasses.verifyConsistent(conf); }
/** * Constructor. * * @param service Service worker * @param configuration Configuration * @param progressable Progressable */ public AbstractEdgeStore( CentralizedServiceWorker<I, V, E> service, ImmutableClassesGiraphConfiguration<I, V, E> configuration, Progressable progressable) { this.service = service; this.configuration = configuration; this.progressable = progressable; transientEdges = new MapMaker() .concurrencyLevel(configuration.getNettyServerExecutionConcurrency()) .makeMap(); reuseEdgeObjects = configuration.reuseEdgeObjects(); useInputOutEdges = configuration.useInputOutEdges(); }
/** * Convert the input edges to the {@link OutEdges} data structure used for computation (if * different). * * @param inputEdges Input edges * @return Compute edges */ private OutEdges<I, E> convertInputToComputeEdges(OutEdges<I, E> inputEdges) { if (!useInputOutEdges) { return inputEdges; } else { return configuration.createAndInitializeOutEdges(inputEdges); } }
@Override public void moveEdgesToVertices() { final boolean createSourceVertex = configuration.getCreateSourceVertex(); if (transientEdges.isEmpty()) { if (LOG.isInfoEnabled()) { LOG.info("moveEdgesToVertices: No edges to move"); } return; } if (LOG.isInfoEnabled()) { LOG.info("moveEdgesToVertices: Moving incoming edges to vertices."); } final BlockingQueue<Integer> partitionIdQueue = new ArrayBlockingQueue<>(transientEdges.size()); partitionIdQueue.addAll(transientEdges.keySet()); int numThreads = configuration.getNumInputSplitsThreads(); CallableFactory<Void> callableFactory = new CallableFactory<Void>() { @Override public Callable<Void> newCallable(int callableId) { return new Callable<Void>() { @Override public Void call() throws Exception { Integer partitionId; I representativeVertexId = configuration.createVertexId(); while ((partitionId = partitionIdQueue.poll()) != null) { Partition<I, V, E> partition = service.getPartitionStore().getOrCreatePartition(partitionId); Map<K, OutEdges<I, E>> partitionEdges = transientEdges.remove(partitionId); Iterator<Et> iterator = getPartitionEdgesIterator(partitionEdges); // process all vertices in given partition while (iterator.hasNext()) { Et entry = iterator.next(); I vertexId = getVertexId(entry, representativeVertexId); OutEdges<I, E> outEdges = convertInputToComputeEdges(removePartitionEdges(entry, partitionEdges)); Vertex<I, V, E> vertex = partition.getVertex(vertexId); // If the source vertex doesn't exist, create it. Otherwise, // just set the edges. if (vertex == null) { if (createSourceVertex) { // createVertex only if it is allowed by configuration vertex = configuration.createVertex(); vertex.initialize( createVertexId(entry), configuration.createVertexValue(), outEdges); partition.putVertex(vertex); } } else { // A vertex may exist with or without edges initially // and optimize the case of no initial edges if (vertex.getNumEdges() == 0) { vertex.setEdges(outEdges); } else { for (Edge<I, E> edge : outEdges) { vertex.addEdge(edge); } } // Some Partition implementations (e.g. ByteArrayPartition) // require us to put back the vertex after modifying it. partition.saveVertex(vertex); } } // Some PartitionStore implementations // (e.g. DiskBackedPartitionStore) require us to put back the // partition after modifying it. service.getPartitionStore().putPartition(partition); } return null; } }; } }; ProgressableUtils.getResultsWithNCallables( callableFactory, numThreads, "move-edges-%d", progressable); // remove all entries transientEdges.clear(); if (LOG.isInfoEnabled()) { LOG.info("moveEdgesToVertices: Finished moving incoming edges to " + "vertices."); } }
/** * Factory method to create a new wrapper over the existing out-edges. * * @param edges Current out-edges * @param conf Configuration * @param <I> Vertex id * @param <E> Edge value * @return The wrapped edges */ public static <I extends WritableComparable, E extends Writable> MutableEdgesWrapper<I, E> wrap( OutEdges<I, E> edges, ImmutableClassesGiraphConfiguration<I, ?, E, ?> conf) { MutableEdgesWrapper<I, E> wrapper = new MutableEdgesWrapper<I, E>(edges, conf.createAndInitializeOutEdges(edges.size())); return wrapper; }