@Test public void shouldWriteToArbitraryRDD() throws Exception { final Configuration configuration = new BaseConfiguration(); configuration.setProperty("spark.master", "local[4]"); configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); configuration.setProperty( Constants.GREMLIN_HADOOP_INPUT_LOCATION, HadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo")); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output"); configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); //////// Graph graph = GraphFactory.open(configuration); graph .compute(SparkGraphComputer.class) .result(GraphComputer.ResultGraph.NEW) .persist(GraphComputer.Persist.EDGES) .program( TraversalVertexProgram.build() .traversal( GraphTraversalSource.build() .engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)), "gremlin-groovy", "g.V()") .create(graph)) .submit() .get(); }
@Test public void shouldWriteToArbitraryRDD() throws Exception { final Configuration configuration = new BaseConfiguration(); configuration.setProperty("spark.master", "local[4]"); configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName()); configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); configuration.setProperty( Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo")); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(this.getClass(), "shouldWriteToArbitraryRDD")); configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); //////// Graph graph = GraphFactory.open(configuration); graph .compute(SparkGraphComputer.class) .result(GraphComputer.ResultGraph.NEW) .persist(GraphComputer.Persist.EDGES) .program( TraversalVertexProgram.build() .traversal( graph.traversal().withComputer(g -> g.compute(SparkGraphComputer.class)), "gremlin-groovy", "g.V()") .create(graph)) .submit() .get(); }
@Test public void shouldPersistRDDBasedOnStorageLevel() throws Exception { Spark.create("local[4]"); int counter = 0; for (final String storageLevel : Arrays.asList("MEMORY_ONLY", "DISK_ONLY", "MEMORY_ONLY_SER", "MEMORY_AND_DISK_SER")) { assertEquals(counter, Spark.getRDDs().size()); assertEquals(counter, Spark.getContext().getPersistentRDDs().size()); counter++; final String rddName = TestHelper.makeTestDataDirectory( PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); configuration.setProperty( Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo")); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_STORAGE_LEVEL, storageLevel); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); Graph graph = GraphFactory.open(configuration); graph .compute(SparkGraphComputer.class) .result(GraphComputer.ResultGraph.NEW) .persist(GraphComputer.Persist.EDGES) .program( TraversalVertexProgram.build() .traversal( graph.traversal().withComputer(SparkGraphComputer.class), "gremlin-groovy", "g.V().groupCount('m').by('name').out()") .create(graph)) .submit() .get(); //////// assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName))); assertEquals( StorageLevel.fromString(storageLevel), Spark.getRDD(Constants.getGraphLocation(rddName)).getStorageLevel()); assertEquals(counter, Spark.getRDDs().size()); assertEquals(counter, Spark.getContext().getPersistentRDDs().size()); } Spark.close(); }
@Test public void shouldNotPersistRDDAcrossJobs() throws Exception { Spark.create("local[4]"); final String rddName = TestHelper.makeTestDataDirectory( PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); configuration.setProperty( Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo")); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty( Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false); // because the spark context is NOT persisted, neither is the RDD Graph graph = GraphFactory.open(configuration); graph .compute(SparkGraphComputer.class) .result(GraphComputer.ResultGraph.NEW) .persist(GraphComputer.Persist.EDGES) .program( TraversalVertexProgram.build() .traversal( graph.traversal().withComputer(SparkGraphComputer.class), "gremlin-groovy", "g.V()") .create(graph)) .submit() .get(); //////// Spark.create("local[4]"); assertFalse(Spark.hasRDD(Constants.getGraphLocation(rddName))); assertEquals(0, Spark.getContext().getPersistentRDDs().size()); Spark.close(); }