/**
* Retrieves the centroids between K-means iterations.
*
* @return the centroids
*/
public static long[] getCentroids() throws IOException {
Configuration conf = setupConf();
FileSystem fs = FileSystem.get(conf);
Path path = new Path(BASE_OUTPUT + CENTROID_FILE);
long[] centroids = new long[4];
FSDataInputStream in = fs.open(path);
centroids[0] = Long.parseLong(in.readUTF());
in.readChar();
in.readUTF();
in.readChar();
centroids[1] = Long.parseLong(in.readUTF());
in.readChar();
in.readUTF();
in.readChar();
in.readUTF();
in.readChar();
centroids[2] = Long.parseLong(in.readUTF());
in.readChar();
in.readUTF();
in.readChar();
in.readUTF();
in.readChar();
centroids[3] = Long.parseLong(in.readUTF());
in.close();
return centroids;
}
private String getJobSummary(FileContext fc, Path path) throws IOException {
Path qPath = fc.makeQualified(path);
FSDataInputStream in = fc.open(qPath);
String jobSummaryString = in.readUTF();
in.close();
return jobSummaryString;
}
/**
* Retrieves the graph root.
*
* @return the graph root
* @throws IOException
*/
public static String getGraphRoot() throws IOException {
Configuration conf = setupConf();
FileSystem fs = FileSystem.get(conf);
FSDataInputStream in = fs.open(new Path(SPATH_OUTPUT + SPATH_ROOTFILE));
String username = in.readUTF();
in.close();
return username;
}
private void loadUnforwardedCounts() throws IOException {
unforwardedCounts = new HashMap<Unforwarded, Long>();
FSDataInputStream in = getHdfs().open(getUnforwardedCountsPath());
while (in.available() > 0) {
String u = in.readUTF();
Long count = in.readLong();
unforwardedCounts.put(Unforwarded.valueOf(u), count);
}
in.close();
}
@org.junit.Test
public void testTextWriteAndRead() throws Exception {
String testString = "Is there anyone out there?";
String readChars = null;
FSDataOutputStream dfsOut = null;
dfsOut = fs.create(new Path("test1.txt"));
dfsOut.writeUTF(testString);
dfsOut.close();
FSDataInputStream dfsin = null;
dfsin = fs.open(new Path("test1.txt"));
readChars = dfsin.readUTF();
dfsin.close();
assertEquals(testString, readChars);
fs.delete(new Path("test1.txt"), true);
assertFalse(fs.exists(new Path("test1")));
}
@SuppressWarnings("unchecked")
public List<KeyData<K>> getKeyData() throws IOException, ClassNotFoundException {
if (keyData != null) return keyData;
final FileSystem fs = FileSystem.get(conf);
keyData = new ArrayList<KeyData<K>>();
final long filesize = fs.getFileStatus(path).getLen();
FSDataInputStream in = fs.open(path);
/* The last long in the file is the start position of the trailer section */
in.seek(filesize - 8);
long metaDataStartPos = in.readLong();
in.seek(metaDataStartPos);
ObjectMapper mapper = new ObjectMapper();
metadataJson = mapper.readValue(in.readUTF(), JsonNode.class);
int keySectionSize = in.readInt();
// load the key section
byte[] keySection = new byte[keySectionSize];
in.seek(filesize - keySectionSize - 8);
in.read(keySection, 0, keySectionSize);
in.close();
ByteArrayInputStream bis = new ByteArrayInputStream(keySection);
DataInput dataInput = new DataInputStream(bis);
int numberOfBlocks = metadataJson.get("numberOfBlocks").getIntValue();
// load the key section
keyClass = (Class<K>) ClassCache.forName(JsonUtils.getText(metadataJson, "keyClass"));
valueClass = (Class<V>) ClassCache.forName(JsonUtils.getText(metadataJson, "valueClass"));
SerializationFactory serializationFactory = new SerializationFactory(conf);
Deserializer<K> deserializer = serializationFactory.getDeserializer(keyClass);
deserializer.open(bis);
while (bis.available() > 0 && numberOfBlocks > 0) {
K key = deserializer.deserialize(null);
long offset = dataInput.readLong();
long blockId = dataInput.readLong();
long numRecords = dataInput.readLong();
keyData.add(new KeyData<K>(key, offset, 0, numRecords, blockId));
numberOfBlocks--;
}
// Assign length to each keydata entry
int numEntries = keyData.size();
for (int i = 1; i < numEntries; i++) {
KeyData<K> prev = keyData.get(i - 1);
KeyData<K> current = keyData.get(i);
prev.setLength(current.getOffset() - prev.getOffset());
}
if (numEntries > 0) {
KeyData<K> last = keyData.get(numEntries - 1);
last.setLength(metaDataStartPos - last.offset);
}
return keyData;
}
public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem hdfs = cluster.getFileSystem();
final String namenode = hdfs.getUri().toString();
FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
// Create two files of the same name, same length but different
// contents
final String testfilename = "test";
final String srcData = "act act act";
final String destData = "cat cat cat";
if (namenode.startsWith("hdfs://")) {
deldir(hdfs, "/logs");
Path srcPath = new Path("/srcdat", testfilename);
Path destPath = new Path("/destdat", testfilename);
FSDataOutputStream out = fs.create(srcPath, true);
out.writeUTF(srcData);
out.close();
out = fs.create(destPath, true);
out.writeUTF(destData);
out.close();
// Run with -skipcrccheck option
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-p",
"-update",
"-skipcrccheck",
"-log",
namenode + "/logs",
namenode + "/srcdat",
namenode + "/destdat"
});
// File should not be overwritten
FSDataInputStream in = hdfs.open(destPath);
String s = in.readUTF();
System.out.println("Dest had: " + s);
assertTrue("Dest got over written even with skip crc", s.equalsIgnoreCase(destData));
in.close();
deldir(hdfs, "/logs");
// Run without the option
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-p",
"-update",
"-log",
namenode + "/logs",
namenode + "/srcdat",
namenode + "/destdat"
});
// File should be overwritten
in = hdfs.open(destPath);
s = in.readUTF();
System.out.println("Dest had: " + s);
assertTrue("Dest did not get overwritten without skip crc", s.equalsIgnoreCase(srcData));
in.close();
deldir(hdfs, "/destdat");
deldir(hdfs, "/srcdat");
deldir(hdfs, "/logs");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
