public static void unCompressLzo(String input, String output, Configuration conf) {
LzopCodec lzo = null;
InputStream is = null;
InputStreamReader isr = null;
BufferedReader reader = null;
String line = null;
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output)));
lzo = new LzopCodec();
lzo.setConf(conf);
is = lzo.createInputStream(new FileInputStream(input));
isr = new InputStreamReader(is);
reader = new BufferedReader(isr);
while ((line = reader.readLine()) != null) {
bw.write(line);
bw.newLine();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (null != reader) reader.close();
if (null != isr) isr.close();
if (null != is) is.close();
if (null != bw) bw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static List<String> readLzoFile(String lzoFilePath, Configuration conf) {
LzopCodec lzo = null;
InputStream is = null;
InputStreamReader isr = null;
BufferedReader reader = null;
List<String> result = null;
String line = null;
try {
lzo = new LzopCodec();
lzo.setConf(conf);
is = lzo.createInputStream(new FileInputStream(lzoFilePath));
isr = new InputStreamReader(is);
reader = new BufferedReader(isr);
result = new ArrayList<String>();
while ((line = reader.readLine()) != null) {
result.add(line);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
}
if (isr != null) {
isr.close();
}
if (is != null) {
is.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
}
public static void write2LzoFile(String destLzoFilePath, Configuration conf, byte[] datas) {
LzopCodec lzo = null;
OutputStream out = null;
try {
lzo = new LzopCodec();
lzo.setConf(conf);
out = lzo.createOutputStream(new FileOutputStream(destLzoFilePath));
out.write(datas);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static boolean copyLocalFilesToOneHDFS(
FileSystem filesystemIn,
List<Path> todoNames,
FileSystem filesystemOut,
Path pathOut,
String filenamePrefix,
Algorithm alg,
Configuration conf)
throws IOException {
int outputBufferSize = 10 * 1024 * 1024;
int inputBufferSize = 10 * 1024 * 1024;
int copyBufferSize = 10 * 1024 * 1024;
// batch load data
FSDataOutputStream out = null;
OutputStream os = null;
try {
String postfix;
if (alg == Algorithm.LZO) {
postfix = ".lzo";
} else if (alg == Algorithm.GZ) {
postfix = ".gz";
} else {
postfix = ".txt";
}
Path fileOut = new Path(pathOut, filenamePrefix + postfix);
if (filesystemOut.exists(fileOut)) {
System.err.println("[WARN] output file exists already! remove it first!" + fileOut);
return false;
}
out = filesystemOut.create(fileOut, true, outputBufferSize);
if (alg == Algorithm.LZO) {
LzopCodec codec = new LzopCodec();
codec.setConf(conf);
os = codec.createOutputStream(out);
} else if (alg == Algorithm.GZ) {
os = new GZIPOutputStream(out);
} else {
os = out;
}
boolean isDone = false;
byte[] buffer = new byte[copyBufferSize];
int idx = 0;
for (Path p : todoNames) {
FSDataInputStream fsdi = null;
int readDataLen = 0;
boolean read = false;
boolean write = false;
try {
System.out.println("[INFO]processing: (" + idx + ")" + p);
fsdi = filesystemIn.open(p, inputBufferSize);
for (; ; ) {
int n = fsdi.read(buffer);
if (n < 0) {
break;
}
read = true;
os.write(buffer, 0, n);
readDataLen += n;
write = true;
}
isDone = true;
} catch (IOException e) {
System.err.println(
"[WARN] error reading :"
+ p.toString()
+ ", read data len: "
+ readDataLen
+ ", hasRead:"
+ read
+ ", hasWrite:"
+ write
+ ", msg:"
+ e.getMessage());
} finally {
if (fsdi != null) {
try {
fsdi.close();
} catch (IOException ignored) {
}
fsdi = null;
idx++;
}
}
if (read && !write) {
break;
}
}
if (!isDone) {
System.err.println("[ERROR] not all files are written!!!");
return true;
}
return true;
} finally {
if (os != null) {
try {
os.close();
} catch (IOException ignored) {
}
} else if (out != null) {
try {
out.close();
} catch (IOException ignored) {
}
}
}
}