@Override
@SuppressWarnings("unchecked")
protected void cleanup(Reducer.Context context) throws IOException, InterruptedException {
if (!Project.getProject().isMetadataCollectStandard()) {
// write summary headers with all metadata, but for standard metadata don't write the last
// line
context.write(new Text("Hash"), new Text(columnMetadata.delimiterSeparatedHeaders()));
}
zipFileWriter.closeZip();
if (Project.getProject().isLuceneIndexEnabled()) {
mergeLuceneIndex();
}
Project project = Project.getProject();
if (project.isEnvHadoop()) {
String outputPath = Project.getProject().getProperty(ParameterProcessing.OUTPUT_DIR_HADOOP);
String zipFileName = zipFileWriter.getZipFileName();
if (project.isFsHdfs()) {
String cmd =
"hadoop fs -copyFromLocal "
+ zipFileName
+ " "
+ outputPath
+ File.separator
+ context.getTaskAttemptID()
+ ".zip";
OsUtil.runCommand(cmd);
} else if (project.isFsS3()) {
S3Agent s3agent = new S3Agent();
String run = project.getRun();
if (!run.isEmpty()) {
run = run + "/";
}
String s3key =
project.getProjectCode()
+ File.separator
+ "output/"
+ run
+ "results/"
+ context.getTaskAttemptID()
+ ".zip";
// Keep updating the hadoop progress
int refreshInterval = 60000;
Timer timer = new Timer(refreshInterval, this);
timer.start();
s3agent.putFileInS3(zipFileName, s3key);
timer.stop();
}
}
Stats.getInstance().setJobFinished();
}
