private List<String> readFileContentAsList(File jobFile) { List<String> lines = new ArrayList<String>(); FileTouchPoint fileTouchPoint = this.fileTouchPointMap.get(jobFile.getAbsolutePath()); boolean needToReadFile = false; long lastReadPoint = -1; if (fileTouchPoint == null) { needToReadFile = true; } else if (fileTouchPoint.shouldReadFile(jobFile.lastModified())) { needToReadFile = true; lastReadPoint = fileTouchPoint.lastReadPoint; } if (needToReadFile) { RandomAccessFile raf = null; try { raf = FileHelper.randomAccessFile(jobFile, "r"); if (lastReadPoint != -1) raf.seek(lastReadPoint); String line = null; while ((line = raf.readLine()) != null) { lines.add(line); } lastReadPoint = raf.getFilePointer(); fileTouchPointMap.put( jobFile.getAbsolutePath(), new FileTouchPoint(jobFile.lastModified(), lastReadPoint)); } catch (FileNotFoundException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } catch (IOException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } finally { if (raf != null) try { FileHelper.close(raf); } catch (IOException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } } } return lines; }
// both reads and writes - need to be synchronized for consistent data to be written public static void deleteWorkflowJob(String workflowJobId) throws IOException { String workflowJobPath = LoaderServerConfiguration.instance().getJobFSConfig().getWorkflowJobPath(workflowJobId); File file = new File(workflowJobPath); if (!file.exists()) throw new RuntimeException("WorkflowJobId " + workflowJobId + " not found."); File statusFile = new File( LoaderServerConfiguration.instance() .getJobFSConfig() .getWorkflowJobStatusPath(workflowJobId)); ScheduledWorkFlowJob workflow = objectMapper.readValue(statusFile, ScheduledWorkFlowJob.class); FileHelper.deleteRecursively(file); File workflowJobsFile = new File( LoaderServerConfiguration.instance() .getJobFSConfig() .getScheduledWorkflowJobsFile(workflow.getWorkflowName())); synchronized (ScheduledWorkFlowJob.class) { ArrayList<String> allWorkflowJobIds = objectMapper.readValue(workflowJobsFile, ArrayList.class); allWorkflowJobIds.remove(workflowJobId); objectMapper.writerWithDefaultPrettyPrinter().writeValue(workflowJobsFile, allWorkflowJobIds); } }
private void crunchJobCounters(String jobId) { List<File> jobFiles = FileHelper.pathFiles(this.jobFSConfig.getJobPath(jobId), true); for (File jobFile : jobFiles) { if (jobFile.getAbsolutePath().contains("cumulative")) { crunchJobFileCounter(jobId, jobFile); } } }
public void persistWorkFlowJob() throws IOException { File workFlowStatusFile = new File(configuration.getJobFSConfig().getWorkflowJobStatusPath(this.workFlowId)); if (!workFlowStatusFile.exists()) FileHelper.createFilePath(workFlowStatusFile.getAbsolutePath()); objectMapper .writerWithDefaultPrettyPrinter() .writeValue(new FileOutputStream(workFlowStatusFile), this); }
public void crunchJobFileCounter(String jobId, File jobFile) { List<String> fileContentLines = readFileContentAsList(jobFile); if (fileContentLines.size() > 0) { String newFile = jobFile.getAbsolutePath().replace("cumulative", FILE_EXTENSION); BufferedWriter bw = null; try { bw = FileHelper.bufferedWriter(newFile, true); LastPoint lastPoint = this.fileLastCrunchPointMap.get(jobFile.getAbsolutePath()); if (lastPoint == null) { String firstContentLine = fileContentLines.remove(0); String[] tokens = firstContentLine.split(","); lastPoint = new LastPoint(Long.parseLong(tokens[0]), Long.parseLong(tokens[1])); } while (fileContentLines.size() > 0) { String currentContentLine = fileContentLines.remove(0); String[] tokens = currentContentLine.split(","); long currentContentTimeMS = Long.parseLong(tokens[0]); long currentContentCount = Long.parseLong(tokens[1]); long opsDone = currentContentCount - lastPoint.count; long timeTakenMS = currentContentTimeMS - lastPoint.time; float timeTakenSec = (float) timeTakenMS / MathConstant.THOUSAND; float tps = opsDone / timeTakenSec; CounterStatsInstance counterStatsInstance = new CounterStatsInstance() .setTime(Clock.dateFromMS(currentContentTimeMS)) .setCount(currentContentCount) .setThroughput(tps); bw.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n"); bw.flush(); BufferedWriter bwLast = FileHelper.bufferedWriter(newFile + ".last", false); bwLast.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n"); bwLast.flush(); lastPoint = new LastPoint(currentContentTimeMS, currentContentCount); this.fileLastCrunchPointMap.put(jobFile.getAbsolutePath(), lastPoint); } } catch (FileNotFoundException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } catch (IOException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } finally { try { FileHelper.close(bw); } catch (IOException e) { e .printStackTrace(); // To change body of catch statement use File | Settings | File // Templates. } } } }