@Override
public void configure(final Map<String, Object> config, final Map<String, Graph> graphs) {
final GryoMapper.Builder builder;
final Object graphToUseForMapper = config.get(TOKEN_USE_MAPPER_FROM_GRAPH);
if (graphToUseForMapper != null) {
if (null == graphs)
throw new IllegalStateException(
String.format(
"No graphs have been provided to the serializer and therefore %s is not a valid configuration",
TOKEN_USE_MAPPER_FROM_GRAPH));
final Graph g = graphs.get(graphToUseForMapper.toString());
if (null == g)
throw new IllegalStateException(
String.format(
"There is no graph named [%s] configured to be used in the %s setting",
graphToUseForMapper, TOKEN_USE_MAPPER_FROM_GRAPH));
// a graph was found so use the mapper it constructs. this allows gryo to be auto-configured
// with any
// custom classes that the implementation allows for
builder = g.io(GryoIo.build()).mapper();
} else {
// no graph was supplied so just use the default - this will likely be the case when using a
// graph
// with no custom classes or a situation where the user needs complete control like when using
// two
// distinct implementations each with their own custom classes.
builder = GryoMapper.build();
}
addIoRegistries(config, builder);
addCustomClasses(config, builder);
this.serializeToString =
Boolean.parseBoolean(
config.getOrDefault(TOKEN_SERIALIZE_RESULT_TO_STRING, "false").toString());
this.gryoMapper = builder.create();
}
@Test
public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throws Exception {
Spark.create("local[4]");
final String rddName =
TestHelper.makeTestDataDirectory(
PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final Configuration readConfiguration = super.getBaseConfiguration();
readConfiguration.setProperty(
Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
readConfiguration.setProperty(
Constants.GREMLIN_HADOOP_INPUT_LOCATION,
SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
readConfiguration.setProperty(
Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
readConfiguration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
readConfiguration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
Graph pageRankGraph = GraphFactory.open(readConfiguration);
///////////////
final Configuration writeConfiguration = new BaseConfiguration();
writeConfiguration.setProperty(Graph.GRAPH, TinkerGraph.class.getCanonicalName());
writeConfiguration.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
writeConfiguration.setProperty(
TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,
TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class)
+ "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
final Graph bulkLoaderGraph =
pageRankGraph
.compute(SparkGraphComputer.class)
.persist(GraphComputer.Persist.EDGES)
.program(PageRankVertexProgram.build().create(pageRankGraph))
.submit()
.get()
.graph();
bulkLoaderGraph
.compute(SparkGraphComputer.class)
.persist(GraphComputer.Persist.NOTHING)
.workers(1)
.program(
BulkLoaderVertexProgram.build()
.userSuppliedIds(true)
.writeGraph(writeConfiguration)
.create(bulkLoaderGraph))
.submit()
.get();
////
Spark.create(readConfiguration);
assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName)));
assertEquals(1, Spark.getContext().getPersistentRDDs().size());
////
final Graph graph = TinkerGraph.open();
final GraphTraversalSource g = graph.traversal();
graph
.io(IoCore.gryo())
.readGraph(
TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class)
+ "testBulkLoaderVertexProgramChainWithInputOutputHelperMapping.kryo");
assertEquals(6l, g.V().count().next().longValue());
assertEquals(6l, g.E().count().next().longValue());
assertEquals("marko", g.V().has("name", "marko").values("name").next());
assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue());
////
Spark.close();
}