private List<String> readFileContentAsList(File jobFile) {
    List<String> lines = new ArrayList<String>();
    FileTouchPoint fileTouchPoint = this.fileTouchPointMap.get(jobFile.getAbsolutePath());

    boolean needToReadFile = false;
    long lastReadPoint = -1;
    if (fileTouchPoint == null) {
      needToReadFile = true;
    } else if (fileTouchPoint.shouldReadFile(jobFile.lastModified())) {
      needToReadFile = true;
      lastReadPoint = fileTouchPoint.lastReadPoint;
    }

    if (needToReadFile) {
      RandomAccessFile raf = null;
      try {
        raf = FileHelper.randomAccessFile(jobFile, "r");

        if (lastReadPoint != -1) raf.seek(lastReadPoint);

        String line = null;
        while ((line = raf.readLine()) != null) {
          lines.add(line);
        }
        lastReadPoint = raf.getFilePointer();
        fileTouchPointMap.put(
            jobFile.getAbsolutePath(), new FileTouchPoint(jobFile.lastModified(), lastReadPoint));

      } catch (FileNotFoundException e) {
        e
            .printStackTrace(); // To change body of catch statement use File | Settings | File
                                // Templates.
      } catch (IOException e) {
        e
            .printStackTrace(); // To change body of catch statement use File | Settings | File
                                // Templates.
      } finally {
        if (raf != null)
          try {
            FileHelper.close(raf);
          } catch (IOException e) {
            e
                .printStackTrace(); // To change body of catch statement use File | Settings | File
                                    // Templates.
          }
      }
    }
    return lines;
  }
示例#2
0
 // both reads and writes - need to be synchronized for consistent data to be written
 public static void deleteWorkflowJob(String workflowJobId) throws IOException {
   String workflowJobPath =
       LoaderServerConfiguration.instance().getJobFSConfig().getWorkflowJobPath(workflowJobId);
   File file = new File(workflowJobPath);
   if (!file.exists())
     throw new RuntimeException("WorkflowJobId " + workflowJobId + " not found.");
   File statusFile =
       new File(
           LoaderServerConfiguration.instance()
               .getJobFSConfig()
               .getWorkflowJobStatusPath(workflowJobId));
   ScheduledWorkFlowJob workflow = objectMapper.readValue(statusFile, ScheduledWorkFlowJob.class);
   FileHelper.deleteRecursively(file);
   File workflowJobsFile =
       new File(
           LoaderServerConfiguration.instance()
               .getJobFSConfig()
               .getScheduledWorkflowJobsFile(workflow.getWorkflowName()));
   synchronized (ScheduledWorkFlowJob.class) {
     ArrayList<String> allWorkflowJobIds =
         objectMapper.readValue(workflowJobsFile, ArrayList.class);
     allWorkflowJobIds.remove(workflowJobId);
     objectMapper.writerWithDefaultPrettyPrinter().writeValue(workflowJobsFile, allWorkflowJobIds);
   }
 }
 private void crunchJobCounters(String jobId) {
   List<File> jobFiles = FileHelper.pathFiles(this.jobFSConfig.getJobPath(jobId), true);
   for (File jobFile : jobFiles) {
     if (jobFile.getAbsolutePath().contains("cumulative")) {
       crunchJobFileCounter(jobId, jobFile);
     }
   }
 }
示例#4
0
 public void persistWorkFlowJob() throws IOException {
   File workFlowStatusFile =
       new File(configuration.getJobFSConfig().getWorkflowJobStatusPath(this.workFlowId));
   if (!workFlowStatusFile.exists())
     FileHelper.createFilePath(workFlowStatusFile.getAbsolutePath());
   objectMapper
       .writerWithDefaultPrettyPrinter()
       .writeValue(new FileOutputStream(workFlowStatusFile), this);
 }
  public void crunchJobFileCounter(String jobId, File jobFile) {
    List<String> fileContentLines = readFileContentAsList(jobFile);

    if (fileContentLines.size() > 0) {
      String newFile = jobFile.getAbsolutePath().replace("cumulative", FILE_EXTENSION);
      BufferedWriter bw = null;
      try {
        bw = FileHelper.bufferedWriter(newFile, true);

        LastPoint lastPoint = this.fileLastCrunchPointMap.get(jobFile.getAbsolutePath());
        if (lastPoint == null) {
          String firstContentLine = fileContentLines.remove(0);
          String[] tokens = firstContentLine.split(",");
          lastPoint = new LastPoint(Long.parseLong(tokens[0]), Long.parseLong(tokens[1]));
        }

        while (fileContentLines.size() > 0) {
          String currentContentLine = fileContentLines.remove(0);
          String[] tokens = currentContentLine.split(",");
          long currentContentTimeMS = Long.parseLong(tokens[0]);
          long currentContentCount = Long.parseLong(tokens[1]);
          long opsDone = currentContentCount - lastPoint.count;
          long timeTakenMS = currentContentTimeMS - lastPoint.time;
          float timeTakenSec = (float) timeTakenMS / MathConstant.THOUSAND;
          float tps = opsDone / timeTakenSec;

          CounterStatsInstance counterStatsInstance =
              new CounterStatsInstance()
                  .setTime(Clock.dateFromMS(currentContentTimeMS))
                  .setCount(currentContentCount)
                  .setThroughput(tps);
          bw.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n");
          bw.flush();

          BufferedWriter bwLast = FileHelper.bufferedWriter(newFile + ".last", false);
          bwLast.write(objectMapper.writeValueAsString(counterStatsInstance) + "\n");
          bwLast.flush();

          lastPoint = new LastPoint(currentContentTimeMS, currentContentCount);
          this.fileLastCrunchPointMap.put(jobFile.getAbsolutePath(), lastPoint);
        }

      } catch (FileNotFoundException e) {
        e
            .printStackTrace(); // To change body of catch statement use File | Settings | File
                                // Templates.
      } catch (IOException e) {
        e
            .printStackTrace(); // To change body of catch statement use File | Settings | File
                                // Templates.
      } finally {
        try {
          FileHelper.close(bw);
        } catch (IOException e) {
          e
              .printStackTrace(); // To change body of catch statement use File | Settings | File
                                  // Templates.
        }
      }
    }
  }