@Test
 public void getTestDb() throws Exception {
   Iterable<String> results;
   GiraphConfiguration conf = new GiraphConfiguration();
   GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
   GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
   GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
   GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
   GIRAPH_GORA_START_KEY.set(conf, "1");
   GIRAPH_GORA_END_KEY.set(conf, "4");
   conf.set(
       "io.serializations",
       "org.apache.hadoop.io.serializer.WritableSerialization,"
           + "org.apache.hadoop.io.serializer.JavaSerialization");
   conf.setComputationClass(EmptyComputation.class);
   conf.setEdgeInputFormatClass(GoraTestEdgeInputFormat.class);
   conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
   results = InternalVertexRunner.run(conf, new String[0], new String[0]);
   Assert.assertNotNull(results);
   Assert.assertEquals(3, ((ArrayList<?>) results).size());
   if (results instanceof Collection<?> & (((Collection<?>) results).size() == 2)) {
     Assert.assertEquals("33\t0.0", ((ArrayList<?>) results).get(0).toString());
     Assert.assertEquals("22\t0.0", ((ArrayList<?>) results).get(1).toString());
     Assert.assertEquals("11\t0.0", ((ArrayList<?>) results).get(2).toString());
   }
 }
  /** Test if aggregators are are handled properly when restarting from a checkpoint */
  @Test
  public void testAggregatorsCheckpointing()
      throws ClassNotFoundException, IOException, InterruptedException {
    Path checkpointsDir = getTempPath("checkPointsForTesting");
    Path outputPath = getTempPath(getCallingMethodName());
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);

    GiraphConfiguration configuration = job.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());
    GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(configuration, false);
    configuration.setCheckpointFrequency(4);

    assertTrue(job.run(true));

    // Restart the test from superstep 4
    System.out.println(
        "testAggregatorsCheckpointing: Restarting from "
            + "superstep 4 with checkpoint path = "
            + checkpointsDir);
    outputPath = getTempPath(getCallingMethodName() + "Restarted");
    conf = new GiraphConfiguration();
    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted", conf, outputPath);
    job.getConfiguration()
        .setMasterComputeClass(SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    GiraphConfiguration restartedJobConf = restartedJob.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJobConf, checkpointsDir.toString());
    restartedJobConf.setLong(GiraphConstants.RESTART_SUPERSTEP, 4);

    assertTrue(restartedJob.run(true));
  }
 /** Tests if aggregators are handled on a proper way during supersteps */
 @Test
 public void testAggregatorsHandling()
     throws IOException, ClassNotFoundException, InterruptedException {
   GiraphConfiguration conf = new GiraphConfiguration();
   conf.setComputationClass(AggregatorsTestComputation.class);
   conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
   conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
   GiraphJob job = prepareJob(getCallingMethodName(), conf);
   job.getConfiguration()
       .setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
   // test with aggregators split in a few requests
   job.getConfiguration().setInt(AggregatorUtils.MAX_BYTES_PER_AGGREGATOR_REQUEST, 50);
   assertTrue(job.run(true));
 }
Exemplo n.º 4
0
  @Test
  public void testCheckJobThrows() throws Exception {
    String tableName = "test1";
    hiveServer.createTable(
        "CREATE TABLE "
            + tableName
            + " (i1 BIGINT, i2 INT) "
            + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
    String[] rows = {
      "1\t2", "2\t3", "2\t4", "4\t1",
    };
    hiveServer.loadData(tableName, rows);

    GiraphConfiguration conf = new GiraphConfiguration();
    HIVE_EDGE_INPUT.setTable(conf, tableName);
    HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
    conf.setComputationClass(ComputationCountEdges.class);
    conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
    assertNull(InternalVertexRunner.run(conf, new String[0], new String[0]));
  }
 @Test
 public void getEmptyDb() throws Exception {
   Iterable<String> results;
   Iterator<String> result;
   GiraphConfiguration conf = new GiraphConfiguration();
   GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
   GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
   GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
   GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
   GIRAPH_GORA_START_KEY.set(conf, "1");
   GIRAPH_GORA_END_KEY.set(conf, "3");
   conf.set(
       "io.serializations",
       "org.apache.hadoop.io.serializer.WritableSerialization,"
           + "org.apache.hadoop.io.serializer.JavaSerialization");
   conf.setComputationClass(EmptyComputation.class);
   conf.setEdgeInputFormatClass(GoraGEdgeEdgeInputFormat.class);
   results = InternalVertexRunner.run(conf, new String[0], new String[0]);
   Assert.assertNotNull(results);
   result = results.iterator();
   Assert.assertFalse(result.hasNext());
 }