public static void write2LzoFile(String destLzoFilePath, Configuration conf, byte[] datas) {
LzopCodec lzo = null;
OutputStream out = null;
try {
lzo = new LzopCodec();
lzo.setConf(conf);
out = lzo.createOutputStream(new FileOutputStream(destLzoFilePath));
out.write(datas);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static boolean copyLocalFilesToOneHDFS(
FileSystem filesystemIn,
List<Path> todoNames,
FileSystem filesystemOut,
Path pathOut,
String filenamePrefix,
Algorithm alg,
Configuration conf)
throws IOException {
int outputBufferSize = 10 * 1024 * 1024;
int inputBufferSize = 10 * 1024 * 1024;
int copyBufferSize = 10 * 1024 * 1024;
// batch load data
FSDataOutputStream out = null;
OutputStream os = null;
try {
String postfix;
if (alg == Algorithm.LZO) {
postfix = ".lzo";
} else if (alg == Algorithm.GZ) {
postfix = ".gz";
} else {
postfix = ".txt";
}
Path fileOut = new Path(pathOut, filenamePrefix + postfix);
if (filesystemOut.exists(fileOut)) {
System.err.println("[WARN] output file exists already! remove it first!" + fileOut);
return false;
}
out = filesystemOut.create(fileOut, true, outputBufferSize);
if (alg == Algorithm.LZO) {
LzopCodec codec = new LzopCodec();
codec.setConf(conf);
os = codec.createOutputStream(out);
} else if (alg == Algorithm.GZ) {
os = new GZIPOutputStream(out);
} else {
os = out;
}
boolean isDone = false;
byte[] buffer = new byte[copyBufferSize];
int idx = 0;
for (Path p : todoNames) {
FSDataInputStream fsdi = null;
int readDataLen = 0;
boolean read = false;
boolean write = false;
try {
System.out.println("[INFO]processing: (" + idx + ")" + p);
fsdi = filesystemIn.open(p, inputBufferSize);
for (; ; ) {
int n = fsdi.read(buffer);
if (n < 0) {
break;
}
read = true;
os.write(buffer, 0, n);
readDataLen += n;
write = true;
}
isDone = true;
} catch (IOException e) {
System.err.println(
"[WARN] error reading :"
+ p.toString()
+ ", read data len: "
+ readDataLen
+ ", hasRead:"
+ read
+ ", hasWrite:"
+ write
+ ", msg:"
+ e.getMessage());
} finally {
if (fsdi != null) {
try {
fsdi.close();
} catch (IOException ignored) {
}
fsdi = null;
idx++;
}
}
if (read && !write) {
break;
}
}
if (!isDone) {
System.err.println("[ERROR] not all files are written!!!");
return true;
}
return true;
} finally {
if (os != null) {
try {
os.close();
} catch (IOException ignored) {
}
} else if (out != null) {
try {
out.close();
} catch (IOException ignored) {
}
}
}
}
