Beispiel #1
0
  /**
   * Retrieves the centroids between K-means iterations.
   *
   * @return the centroids
   */
  public static long[] getCentroids() throws IOException {
    Configuration conf = setupConf();
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(BASE_OUTPUT + CENTROID_FILE);
    long[] centroids = new long[4];

    FSDataInputStream in = fs.open(path);
    centroids[0] = Long.parseLong(in.readUTF());
    in.readChar();

    in.readUTF();
    in.readChar();
    centroids[1] = Long.parseLong(in.readUTF());
    in.readChar();
    in.readUTF();
    in.readChar();

    in.readUTF();
    in.readChar();
    centroids[2] = Long.parseLong(in.readUTF());
    in.readChar();
    in.readUTF();
    in.readChar();

    in.readUTF();
    in.readChar();
    centroids[3] = Long.parseLong(in.readUTF());

    in.close();

    return centroids;
  }
 private String getJobSummary(FileContext fc, Path path) throws IOException {
   Path qPath = fc.makeQualified(path);
   FSDataInputStream in = fc.open(qPath);
   String jobSummaryString = in.readUTF();
   in.close();
   return jobSummaryString;
 }
Beispiel #3
0
 /**
  * Retrieves the graph root.
  *
  * @return the graph root
  * @throws IOException
  */
 public static String getGraphRoot() throws IOException {
   Configuration conf = setupConf();
   FileSystem fs = FileSystem.get(conf);
   FSDataInputStream in = fs.open(new Path(SPATH_OUTPUT + SPATH_ROOTFILE));
   String username = in.readUTF();
   in.close();
   return username;
 }
  private void loadUnforwardedCounts() throws IOException {

    unforwardedCounts = new HashMap<Unforwarded, Long>();

    FSDataInputStream in = getHdfs().open(getUnforwardedCountsPath());

    while (in.available() > 0) {

      String u = in.readUTF();

      Long count = in.readLong();

      unforwardedCounts.put(Unforwarded.valueOf(u), count);
    }

    in.close();
  }
  @org.junit.Test
  public void testTextWriteAndRead() throws Exception {

    String testString = "Is there anyone out there?";
    String readChars = null;

    FSDataOutputStream dfsOut = null;
    dfsOut = fs.create(new Path("test1.txt"));
    dfsOut.writeUTF(testString);
    dfsOut.close();

    FSDataInputStream dfsin = null;

    dfsin = fs.open(new Path("test1.txt"));
    readChars = dfsin.readUTF();
    dfsin.close();

    assertEquals(testString, readChars);

    fs.delete(new Path("test1.txt"), true);

    assertFalse(fs.exists(new Path("test1")));
  }
  @SuppressWarnings("unchecked")
  public List<KeyData<K>> getKeyData() throws IOException, ClassNotFoundException {
    if (keyData != null) return keyData;

    final FileSystem fs = FileSystem.get(conf);
    keyData = new ArrayList<KeyData<K>>();

    final long filesize = fs.getFileStatus(path).getLen();
    FSDataInputStream in = fs.open(path);

    /* The last long in the file is the start position of the trailer section */
    in.seek(filesize - 8);
    long metaDataStartPos = in.readLong();

    in.seek(metaDataStartPos);

    ObjectMapper mapper = new ObjectMapper();
    metadataJson = mapper.readValue(in.readUTF(), JsonNode.class);

    int keySectionSize = in.readInt();

    // load the key section
    byte[] keySection = new byte[keySectionSize];

    in.seek(filesize - keySectionSize - 8);
    in.read(keySection, 0, keySectionSize);
    in.close();

    ByteArrayInputStream bis = new ByteArrayInputStream(keySection);
    DataInput dataInput = new DataInputStream(bis);

    int numberOfBlocks = metadataJson.get("numberOfBlocks").getIntValue();

    // load the key section
    keyClass = (Class<K>) ClassCache.forName(JsonUtils.getText(metadataJson, "keyClass"));
    valueClass = (Class<V>) ClassCache.forName(JsonUtils.getText(metadataJson, "valueClass"));

    SerializationFactory serializationFactory = new SerializationFactory(conf);
    Deserializer<K> deserializer = serializationFactory.getDeserializer(keyClass);

    deserializer.open(bis);

    while (bis.available() > 0 && numberOfBlocks > 0) {
      K key = deserializer.deserialize(null);

      long offset = dataInput.readLong();
      long blockId = dataInput.readLong();
      long numRecords = dataInput.readLong();

      keyData.add(new KeyData<K>(key, offset, 0, numRecords, blockId));
      numberOfBlocks--;
    }

    // Assign length to each keydata entry
    int numEntries = keyData.size();
    for (int i = 1; i < numEntries; i++) {
      KeyData<K> prev = keyData.get(i - 1);
      KeyData<K> current = keyData.get(i);

      prev.setLength(current.getOffset() - prev.getOffset());
    }

    if (numEntries > 0) {
      KeyData<K> last = keyData.get(numEntries - 1);
      last.setLength(metaDataStartPos - last.offset);
    }

    return keyData;
  }
Beispiel #7
0
  public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
    MiniDFSCluster cluster = null;
    try {
      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      final String namenode = hdfs.getUri().toString();

      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      // Create two files of the same name, same length but different
      // contents
      final String testfilename = "test";
      final String srcData = "act act act";
      final String destData = "cat cat cat";

      if (namenode.startsWith("hdfs://")) {
        deldir(hdfs, "/logs");

        Path srcPath = new Path("/srcdat", testfilename);
        Path destPath = new Path("/destdat", testfilename);
        FSDataOutputStream out = fs.create(srcPath, true);
        out.writeUTF(srcData);
        out.close();

        out = fs.create(destPath, true);
        out.writeUTF(destData);
        out.close();

        // Run with -skipcrccheck option
        ToolRunner.run(
            new DistCpV1(conf),
            new String[] {
              "-p",
              "-update",
              "-skipcrccheck",
              "-log",
              namenode + "/logs",
              namenode + "/srcdat",
              namenode + "/destdat"
            });

        // File should not be overwritten
        FSDataInputStream in = hdfs.open(destPath);
        String s = in.readUTF();
        System.out.println("Dest had: " + s);
        assertTrue("Dest got over written even with skip crc", s.equalsIgnoreCase(destData));
        in.close();

        deldir(hdfs, "/logs");

        // Run without the option
        ToolRunner.run(
            new DistCpV1(conf),
            new String[] {
              "-p",
              "-update",
              "-log",
              namenode + "/logs",
              namenode + "/srcdat",
              namenode + "/destdat"
            });

        // File should be overwritten
        in = hdfs.open(destPath);
        s = in.readUTF();
        System.out.println("Dest had: " + s);

        assertTrue("Dest did not get overwritten without skip crc", s.equalsIgnoreCase(srcData));
        in.close();

        deldir(hdfs, "/destdat");
        deldir(hdfs, "/srcdat");
        deldir(hdfs, "/logs");
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
    }
  }