@Test public void testKinesisStream() { // Tests the API, does not actually test data receiving JavaDStream<byte[]> kinesisStream = KinesisUtils.createStream( ssc, "mySparkStream", "https://kinesis.us-west-2.amazonaws.com", new Duration(2000), InitialPositionInStream.LATEST, StorageLevel.MEMORY_AND_DISK_2()); ssc.stop(); }
@Test public void shouldPersistRDDBasedOnStorageLevel() throws Exception { Spark.create("local[4]"); int counter = 0; for (final String storageLevel : Arrays.asList("MEMORY_ONLY", "DISK_ONLY", "MEMORY_ONLY_SER", "MEMORY_AND_DISK_SER")) { assertEquals(counter, Spark.getRDDs().size()); assertEquals(counter, Spark.getContext().getPersistentRDDs().size()); counter++; final String rddName = TestHelper.makeTestDataDirectory( PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); configuration.setProperty( Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo")); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty( Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_STORAGE_LEVEL, storageLevel); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); Graph graph = GraphFactory.open(configuration); graph .compute(SparkGraphComputer.class) .result(GraphComputer.ResultGraph.NEW) .persist(GraphComputer.Persist.EDGES) .program( TraversalVertexProgram.build() .traversal( graph.traversal().withComputer(SparkGraphComputer.class), "gremlin-groovy", "g.V().groupCount('m').by('name').out()") .create(graph)) .submit() .get(); //////// assertTrue(Spark.hasRDD(Constants.getGraphLocation(rddName))); assertEquals( StorageLevel.fromString(storageLevel), Spark.getRDD(Constants.getGraphLocation(rddName)).getStorageLevel()); assertEquals(counter, Spark.getRDDs().size()); assertEquals(counter, Spark.getContext().getPersistentRDDs().size()); } Spark.close(); }
public CustomHttpReceiver() { super(StorageLevel.MEMORY_AND_DISK_2()); }
public Voter(String host_, int port_) { super(StorageLevel.MEMORY_AND_DISK_2()); host = host_; port = port_; }