@Test
public void getTestDb() throws Exception {
Iterable<String> results;
GiraphConfiguration conf = new GiraphConfiguration();
GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
GIRAPH_GORA_START_KEY.set(conf, "1");
GIRAPH_GORA_END_KEY.set(conf, "4");
conf.set(
"io.serializations",
"org.apache.hadoop.io.serializer.WritableSerialization,"
+ "org.apache.hadoop.io.serializer.JavaSerialization");
conf.setComputationClass(EmptyComputation.class);
conf.setEdgeInputFormatClass(GoraTestEdgeInputFormat.class);
conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
results = InternalVertexRunner.run(conf, new String[0], new String[0]);
Assert.assertNotNull(results);
Assert.assertEquals(3, ((ArrayList<?>) results).size());
if (results instanceof Collection<?> & (((Collection<?>) results).size() == 2)) {
Assert.assertEquals("33\t0.0", ((ArrayList<?>) results).get(0).toString());
Assert.assertEquals("22\t0.0", ((ArrayList<?>) results).get(1).toString());
Assert.assertEquals("11\t0.0", ((ArrayList<?>) results).get(2).toString());
}
}
/** Test if aggregators are are handled properly when restarting from a checkpoint */
@Test
public void testAggregatorsCheckpointing()
throws ClassNotFoundException, IOException, InterruptedException {
Path checkpointsDir = getTempPath("checkPointsForTesting");
Path outputPath = getTempPath(getCallingMethodName());
GiraphConfiguration conf = new GiraphConfiguration();
conf.setComputationClass(AggregatorsTestComputation.class);
conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
GiraphConfiguration configuration = job.getConfiguration();
GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());
GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(configuration, false);
configuration.setCheckpointFrequency(4);
assertTrue(job.run(true));
// Restart the test from superstep 4
System.out.println(
"testAggregatorsCheckpointing: Restarting from "
+ "superstep 4 with checkpoint path = "
+ checkpointsDir);
outputPath = getTempPath(getCallingMethodName() + "Restarted");
conf = new GiraphConfiguration();
conf.setComputationClass(AggregatorsTestComputation.class);
conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted", conf, outputPath);
job.getConfiguration()
.setMasterComputeClass(SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
GiraphConfiguration restartedJobConf = restartedJob.getConfiguration();
GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJobConf, checkpointsDir.toString());
restartedJobConf.setLong(GiraphConstants.RESTART_SUPERSTEP, 4);
assertTrue(restartedJob.run(true));
}
/** Tests if aggregators are handled on a proper way during supersteps */
@Test
public void testAggregatorsHandling()
throws IOException, ClassNotFoundException, InterruptedException {
GiraphConfiguration conf = new GiraphConfiguration();
conf.setComputationClass(AggregatorsTestComputation.class);
conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
GiraphJob job = prepareJob(getCallingMethodName(), conf);
job.getConfiguration()
.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
// test with aggregators split in a few requests
job.getConfiguration().setInt(AggregatorUtils.MAX_BYTES_PER_AGGREGATOR_REQUEST, 50);
assertTrue(job.run(true));
}
@Test
public void testCheckJobThrows() throws Exception {
String tableName = "test1";
hiveServer.createTable(
"CREATE TABLE "
+ tableName
+ " (i1 BIGINT, i2 INT) "
+ " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
String[] rows = {
"1\t2", "2\t3", "2\t4", "4\t1",
};
hiveServer.loadData(tableName, rows);
GiraphConfiguration conf = new GiraphConfiguration();
HIVE_EDGE_INPUT.setTable(conf, tableName);
HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
conf.setComputationClass(ComputationCountEdges.class);
conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
assertNull(InternalVertexRunner.run(conf, new String[0], new String[0]));
}
@Test
public void getEmptyDb() throws Exception {
Iterable<String> results;
Iterator<String> result;
GiraphConfiguration conf = new GiraphConfiguration();
GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
GIRAPH_GORA_START_KEY.set(conf, "1");
GIRAPH_GORA_END_KEY.set(conf, "3");
conf.set(
"io.serializations",
"org.apache.hadoop.io.serializer.WritableSerialization,"
+ "org.apache.hadoop.io.serializer.JavaSerialization");
conf.setComputationClass(EmptyComputation.class);
conf.setEdgeInputFormatClass(GoraGEdgeEdgeInputFormat.class);
results = InternalVertexRunner.run(conf, new String[0], new String[0]);
Assert.assertNotNull(results);
result = results.iterator();
Assert.assertFalse(result.hasNext());
}