private List<String> readFileContentAsList(File jobFile) {
List<String> lines = new ArrayList<String>();
FileTouchPoint fileTouchPoint = this.fileTouchPointMap.get(jobFile.getAbsolutePath());
boolean needToReadFile = false;
long lastReadPoint = -1;
if (fileTouchPoint == null) {
needToReadFile = true;
} else if (fileTouchPoint.shouldReadFile(jobFile.lastModified())) {
needToReadFile = true;
lastReadPoint = fileTouchPoint.lastReadPoint;
}
if (needToReadFile) {
RandomAccessFile raf = null;
try {
raf = FileHelper.randomAccessFile(jobFile, "r");
if (lastReadPoint != -1) raf.seek(lastReadPoint);
String line = null;
while ((line = raf.readLine()) != null) {
lines.add(line);
}
lastReadPoint = raf.getFilePointer();
fileTouchPointMap.put(
jobFile.getAbsolutePath(), new FileTouchPoint(jobFile.lastModified(), lastReadPoint));
} catch (FileNotFoundException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} catch (IOException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} finally {
if (raf != null)
try {
FileHelper.close(raf);
} catch (IOException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
}
return lines;
}
// both reads and writes - need to be synchronized for consistent data to be written
public static void deleteWorkflowJob(String workflowJobId) throws IOException {
String workflowJobPath =
LoaderServerConfiguration.instance().getJobFSConfig().getWorkflowJobPath(workflowJobId);
File file = new File(workflowJobPath);
if (!file.exists())
throw new RuntimeException("WorkflowJobId " + workflowJobId + " not found.");
File statusFile =
new File(
LoaderServerConfiguration.instance()
.getJobFSConfig()
.getWorkflowJobStatusPath(workflowJobId));
ScheduledWorkFlowJob workflow = objectMapper.readValue(statusFile, ScheduledWorkFlowJob.class);
FileHelper.deleteRecursively(file);
File workflowJobsFile =
new File(
LoaderServerConfiguration.instance()
.getJobFSConfig()
.getScheduledWorkflowJobsFile(workflow.getWorkflowName()));
synchronized (ScheduledWorkFlowJob.class) {
ArrayList<String> allWorkflowJobIds =
objectMapper.readValue(workflowJobsFile, ArrayList.class);
allWorkflowJobIds.remove(workflowJobId);
objectMapper.writerWithDefaultPrettyPrinter().writeValue(workflowJobsFile, allWorkflowJobIds);
}
}
private void crunchJobCounters(String jobId) {
List<File> jobFiles = FileHelper.pathFiles(this.jobFSConfig.getJobPath(jobId), true);
for (File jobFile : jobFiles) {
if (jobFile.getAbsolutePath().contains("cumulative")) {
crunchJobFileCounter(jobId, jobFile);
}
}
}
public void persistWorkFlowJob() throws IOException {
File workFlowStatusFile =
new File(configuration.getJobFSConfig().getWorkflowJobStatusPath(this.workFlowId));
if (!workFlowStatusFile.exists())
FileHelper.createFilePath(workFlowStatusFile.getAbsolutePath());
objectMapper
.writerWithDefaultPrettyPrinter()
.writeValue(new FileOutputStream(workFlowStatusFile), this);
}
public void crunchJobFileCounter(String jobId, File jobFile) {
List<String> fileContentLines = readFileContentAsList(jobFile);
if (fileContentLines.size() > 0) {
String newFile = jobFile.getAbsolutePath().replace("cumulative", FILE_EXTENSION);
BufferedWriter bw = null;
try {
bw = FileHelper.bufferedWriter(newFile, true);
LastPoint lastPoint = this.fileLastCrunchPointMap.get(jobFile.getAbsolutePath());
if (lastPoint == null) {
String firstContentLine = fileContentLines.remove(0);
String[] tokens = firstContentLine.split(",");
lastPoint = new LastPoint(Long.parseLong(tokens[0]), Long.parseLong(tokens[1]));
}
while (fileContentLines.size() > 0) {
String currentContentLine = fileContentLines.remove(0);
String[] tokens = currentContentLine.split(",");
long currentContentTimeMS = Long.parseLong(tokens[0]);
long currentContentCount = Long.parseLong(tokens[1]);
long opsDone = currentContentCount - lastPoint.count;
long timeTakenMS = currentContentTimeMS - lastPoint.time;
float timeTakenSec = (float) timeTakenMS / MathConstant.THOUSAND;
float tps = opsDone / timeTakenSec;
CounterStatsInstance counterStatsInstance =
new CounterStatsInstance()
.setTime(Clock.dateFromMS(currentContentTimeMS))
.setCount(currentContentCount)
.setThroughput(tps);
bw.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n");
bw.flush();
BufferedWriter bwLast = FileHelper.bufferedWriter(newFile + ".last", false);
bwLast.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n");
bwLast.flush();
lastPoint = new LastPoint(currentContentTimeMS, currentContentCount);
this.fileLastCrunchPointMap.put(jobFile.getAbsolutePath(), lastPoint);
}
} catch (FileNotFoundException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} catch (IOException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
} finally {
try {
FileHelper.close(bw);
} catch (IOException e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
}
}
