コード例 #1
0
 @Test
 public void getTestDb() throws Exception {
   Iterable<String> results;
   GiraphConfiguration conf = new GiraphConfiguration();
   GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
   GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
   GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
   GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
   GIRAPH_GORA_START_KEY.set(conf, "1");
   GIRAPH_GORA_END_KEY.set(conf, "4");
   conf.set(
       "io.serializations",
       "org.apache.hadoop.io.serializer.WritableSerialization,"
           + "org.apache.hadoop.io.serializer.JavaSerialization");
   conf.setComputationClass(EmptyComputation.class);
   conf.setEdgeInputFormatClass(GoraTestEdgeInputFormat.class);
   conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
   results = InternalVertexRunner.run(conf, new String[0], new String[0]);
   Assert.assertNotNull(results);
   Assert.assertEquals(3, ((ArrayList<?>) results).size());
   if (results instanceof Collection<?> & (((Collection<?>) results).size() == 2)) {
     Assert.assertEquals("33\t0.0", ((ArrayList<?>) results).get(0).toString());
     Assert.assertEquals("22\t0.0", ((ArrayList<?>) results).get(1).toString());
     Assert.assertEquals("11\t0.0", ((ArrayList<?>) results).get(2).toString());
   }
 }
コード例 #2
0
 /** Tests if aggregators are handled on a proper way during supersteps */
 @Test
 public void testAggregatorsHandling()
     throws IOException, ClassNotFoundException, InterruptedException {
   GiraphConfiguration conf = new GiraphConfiguration();
   conf.setComputationClass(AggregatorsTestComputation.class);
   conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
   conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
   GiraphJob job = prepareJob(getCallingMethodName(), conf);
   job.getConfiguration()
       .setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
   // test with aggregators split in a few requests
   job.getConfiguration().setInt(AggregatorUtils.MAX_BYTES_PER_AGGREGATOR_REQUEST, 50);
   assertTrue(job.run(true));
 }
コード例 #3
0
  @Test
  public void testCheckJobThrows() throws Exception {
    String tableName = "test1";
    hiveServer.createTable(
        "CREATE TABLE "
            + tableName
            + " (i1 BIGINT, i2 INT) "
            + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'");
    String[] rows = {
      "1\t2", "2\t3", "2\t4", "4\t1",
    };
    hiveServer.loadData(tableName, rows);

    GiraphConfiguration conf = new GiraphConfiguration();
    HIVE_EDGE_INPUT.setTable(conf, tableName);
    HIVE_EDGE_INPUT.setClass(conf, HiveIntNullEdge.class);
    conf.setComputationClass(ComputationCountEdges.class);
    conf.setEdgeInputFormatClass(HiveEdgeInputFormat.class);
    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
    assertNull(InternalVertexRunner.run(conf, new String[0], new String[0]));
  }
コード例 #4
0
 @Test
 public void getEmptyDb() throws Exception {
   Iterable<String> results;
   Iterator<String> result;
   GiraphConfiguration conf = new GiraphConfiguration();
   GIRAPH_GORA_DATASTORE_CLASS.set(conf, "org.apache.gora.memory.store.MemStore");
   GIRAPH_GORA_KEYS_FACTORY_CLASS.set(conf, "org.apache.giraph.io.gora.utils.DefaultKeyFactory");
   GIRAPH_GORA_KEY_CLASS.set(conf, "java.lang.String");
   GIRAPH_GORA_PERSISTENT_CLASS.set(conf, "org.apache.giraph.io.gora.generated.GEdge");
   GIRAPH_GORA_START_KEY.set(conf, "1");
   GIRAPH_GORA_END_KEY.set(conf, "3");
   conf.set(
       "io.serializations",
       "org.apache.hadoop.io.serializer.WritableSerialization,"
           + "org.apache.hadoop.io.serializer.JavaSerialization");
   conf.setComputationClass(EmptyComputation.class);
   conf.setEdgeInputFormatClass(GoraGEdgeEdgeInputFormat.class);
   results = InternalVertexRunner.run(conf, new String[0], new String[0]);
   Assert.assertNotNull(results);
   result = results.iterator();
   Assert.assertFalse(result.hasNext());
 }
コード例 #5
0
  /** Test if aggregators are are handled properly when restarting from a checkpoint */
  @Test
  public void testAggregatorsCheckpointing()
      throws ClassNotFoundException, IOException, InterruptedException {
    Path checkpointsDir = getTempPath("checkPointsForTesting");
    Path outputPath = getTempPath(getCallingMethodName());
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);

    GiraphConfiguration configuration = job.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());
    GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(configuration, false);
    configuration.setCheckpointFrequency(4);

    assertTrue(job.run(true));

    // Restart the test from superstep 4
    System.out.println(
        "testAggregatorsCheckpointing: Restarting from "
            + "superstep 4 with checkpoint path = "
            + checkpointsDir);
    outputPath = getTempPath(getCallingMethodName() + "Restarted");
    conf = new GiraphConfiguration();
    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setMasterComputeClass(AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted", conf, outputPath);
    job.getConfiguration()
        .setMasterComputeClass(SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    GiraphConfiguration restartedJobConf = restartedJob.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJobConf, checkpointsDir.toString());
    restartedJobConf.setLong(GiraphConstants.RESTART_SUPERSTEP, 4);

    assertTrue(restartedJob.run(true));
  }
コード例 #6
0
  @Test
  public void testHostRank() throws Exception {

    if (System.getProperty("prop.mapred.job.tracker") != null) {
      if (LOG.isInfoEnabled())
        LOG.info("testHBaseInputOutput: Ignore this test if not local mode.");
      return;
    }

    File jarTest = new File(System.getProperty("prop.jarLocation"));
    if (!jarTest.exists()) {
      fail(
          "Could not find Giraph jar at "
              + "location specified by 'prop.jarLocation'. "
              + "Make sure you built the main Giraph artifact?.");
    }

    MiniHBaseCluster cluster = null;
    MiniZooKeeperCluster zkCluster = null;
    FileSystem fs = null;

    try {
      // using the restart method allows us to avoid having the hbase
      // root directory overwritten by /home/$username
      zkCluster = testUtil.startMiniZKCluster();
      testUtil.restartHBaseCluster(2);
      cluster = testUtil.getMiniHBaseCluster();

      final byte[] OL_BYTES = Bytes.toBytes("ol");
      final byte[] S_BYTES = Bytes.toBytes("s");
      final byte[] METADATA_BYTES = Bytes.toBytes("mtdt");
      final byte[] HR_BYTES = Bytes.toBytes("_hr_");
      final byte[] TAB = Bytes.toBytes(TABLE_NAME);

      Configuration conf = cluster.getConfiguration();
      HTableDescriptor desc = new HTableDescriptor(TAB);
      desc.addFamily(new HColumnDescriptor(OL_BYTES));
      desc.addFamily(new HColumnDescriptor(S_BYTES));
      desc.addFamily(new HColumnDescriptor(METADATA_BYTES));
      HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
      if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
        hbaseAdmin.disableTable(TABLE_NAME);
        hbaseAdmin.deleteTable(TABLE_NAME);
      }
      hbaseAdmin.createTable(desc);

      /**
       * Enter the initial data (a,b), (b,c), (a,c) a = 1.0 - google b = 1.0 - yahoo c = 1.0 - bing
       */
      HTable table = new HTable(conf, TABLE_NAME);

      Put p1 = new Put(Bytes.toBytes("com.google.www"));
      p1.add(OL_BYTES, Bytes.toBytes("www.yahoo.com"), Bytes.toBytes("ab"));

      Put p2 = new Put(Bytes.toBytes("com.google.www"));
      p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("ac"));
      p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("invalid1"));
      p2.add(OL_BYTES, Bytes.toBytes("www.google.com"), Bytes.toBytes("invalid2"));

      Put p3 = new Put(Bytes.toBytes("com.yahoo.www"));
      p3.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("bc"));
      // p3.add(OL_BYTES, Bytes.toBytes(""), Bytes.toBytes("invalid4"));

      Put p4 = new Put(Bytes.toBytes("com.bing.www"));
      // TODO: Handle below case. use apache isValid method.
      p4.add(OL_BYTES, Bytes.toBytes("http://invalidurl"), Bytes.toBytes("invalid5"));
      p4.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));

      Put p5 = new Put(Bytes.toBytes("dummy"));
      p5.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));

      table.put(p1);
      table.put(p2);
      table.put(p3);
      table.put(p4);
      table.put(p5);

      // Set Giraph configuration
      // now operate over HBase using Vertex I/O formats
      conf.set(TableInputFormat.INPUT_TABLE, TABLE_NAME);
      conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE_NAME);

      // Start the giraph job
      GiraphJob giraphJob = new GiraphJob(conf, BspCase.getCallingMethodName());
      GiraphConfiguration giraphConf = giraphJob.getConfiguration();
      giraphConf.setZooKeeperConfiguration(cluster.getMaster().getZooKeeper().getQuorum());
      setupConfiguration(giraphJob);
      giraphConf.setComputationClass(LinkRankComputation.class);
      giraphConf.setMasterComputeClass(LinkRankVertexMasterCompute.class);
      giraphConf.setOutEdgesClass(ByteArrayEdges.class);
      giraphConf.setVertexInputFormatClass(Nutch2HostInputFormat.class);
      giraphConf.setVertexOutputFormatClass(Nutch2HostOutputFormat.class);
      giraphConf.setInt("giraph.linkRank.superstepCount", 10);
      giraphConf.setInt("giraph.linkRank.scale", 10);
      giraphConf.set("giraph.linkRank.family", "mtdt");
      giraphConf.set("giraph.linkRank.qualifier", "_hr_");
      giraphConf.setVertexInputFilterClass(HostRankVertexFilter.class);
      assertTrue(giraphJob.run(true));

      if (LOG.isInfoEnabled()) LOG.info("Giraph job successful. Checking output qualifier.");

      /** Check the results * */
      Result result;
      String key;
      byte[] calculatedScoreByte;
      HashMap expectedValues = new HashMap<String, Double>();
      expectedValues.put("com.google.www", 1.3515060339386287d);
      expectedValues.put("com.yahoo.www", 4.144902009567587d);
      expectedValues.put("com.bing.www", 9.063893290511482d);

      for (Object keyObject : expectedValues.keySet()) {
        key = keyObject.toString();
        result = table.get(new Get(key.getBytes()));
        calculatedScoreByte = result.getValue(METADATA_BYTES, HR_BYTES);
        assertNotNull(calculatedScoreByte);
        assertTrue(calculatedScoreByte.length > 0);
        Assert.assertEquals(
            "Scores are not the same",
            (Double) expectedValues.get(key),
            Bytes.toDouble(calculatedScoreByte),
            DELTA);
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
      if (zkCluster != null) {
        zkCluster.shutdown();
      }
      // clean test files
      if (fs != null) {
        fs.delete(hbaseRootdir);
      }
    }
  }
コード例 #7
0
ファイル: ConnectionTest.java プロジェクト: simon0227/dgraph
 @Before
 public void setUp() {
   GiraphConfiguration tmpConfig = new GiraphConfiguration();
   tmpConfig.setVertexClass(IntVertex.class);
   conf = new ImmutableClassesGiraphConfiguration(tmpConfig);
 }