protected DirectoryScanner createPartition(int partitionIndex, int partitionCount) {
   DirectoryScanner that = new DirectoryScanner();
   that.filePatternRegexp = this.filePatternRegexp;
   that.regex = this.regex;
   that.partitionIndex = partitionIndex;
   that.partitionCount = partitionCount;
   return that;
 }
Пример #2
0
  /**
   * Returns list of mapped files, that should be transformed. Files can by specified via attributes
   * (srcFile, srcDir) or resources (FileSet, FileList, DirSet, etc). Mapped file represents input
   * and output file for transformation.
   *
   * @return list of mapped files
   */
  public List<MappedFile> getMappedFiles() {
    mappedFiles.clear();

    // one src file
    if (getSrcFile() != null) {
      addMappedFile(getSrcFile());
    }

    if (getSrcDir() != null) {
      addMappedFile(getSrcDir());
    }

    Iterator element = resources.iterator();
    while (element.hasNext()) {
      ResourceCollection rc = (ResourceCollection) element.next();
      if (rc instanceof FileSet && rc.isFilesystemOnly()) {
        FileSet fs = (FileSet) rc;
        File fromDir = fs.getDir(getProject());

        DirectoryScanner ds;
        try {
          ds = fs.getDirectoryScanner(getProject());
        } catch (BuildException ex) {
          log("Could not scan directory " + fromDir, ex, Project.MSG_ERR);
          continue;
        }

        for (String f : ds.getIncludedFiles()) {
          addMappedFile(new File(fromDir + System.getProperty("file.separator") + f), fromDir);
        }
      } else {
        if (!rc.isFilesystemOnly()) {
          log("Only filesystem resources are supported", Project.MSG_WARN);
          continue;
        }
        Iterator rcIt = rc.iterator();
        while (rcIt.hasNext()) {
          Resource r = (Resource) rcIt.next();
          if (!r.isExists()) {
            log("Could not find resource " + r.toLongString(), Project.MSG_VERBOSE);
            continue;
          }

          if (r instanceof FileResource) {
            FileResource fr = (FileResource) r;
            addMappedFile(fr.getFile(), fr.getBaseDir());
          } else {
            log(
                "Only file resources are supported (" + r.getClass().getSimpleName() + " found)",
                Project.MSG_WARN);
            continue;
          }
        }
      }
    }

    return mappedFiles;
  }
Пример #3
0
  /**
   * Protected method for simplyfied addinng of mapped files
   *
   * @param file file to add
   * @param baseDir base directory for file
   */
  protected void addMappedFile(File file, File baseDir) {
    if (file.isFile()) {
      if (baseDir == null) {
        baseDir = file.getParentFile();
      }
      MappedFile mappedFile = new MappedFile();
      mappedFile.setFrom(file);
      String filename = file.getName();

      String[] names = getMapper().mapFileName(filename);
      // we don't use original filename if no mapping is available
      if (names == null || names.length == 0) {
        return;
      }

      File newFile = new File(file.getParent() + System.getProperty("file.separator") + names[0]);
      if (getDestDir() != null) {
        try {
          newFile =
              new File(
                  newFile
                      .getCanonicalPath()
                      .replace(baseDir.getCanonicalPath(), getDestDir().getCanonicalPath()));
        } catch (IOException ex) {
          log("Couldn't map file", ex, Project.MSG_WARN);
          return;
        }
      }
      mappedFile.setTo(newFile);
      mappedFiles.add(mappedFile);
    } else if (file.isDirectory()) {
      if (baseDir == null) {
        baseDir = file;
      }
      DirectoryScanner ds = new DirectoryScanner();
      ds.setBasedir(file);
      ds.scan();

      for (String fileName : ds.getIncludedFiles()) {
        addMappedFile(new File(file + System.getProperty("file.separator") + fileName), baseDir);
      }
    }
  }
  /** Scans the directory for new files. */
  protected void scanDirectory() {
    if (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis) {
      Set<Path> newPaths = scanner.scan(fs, filePath, processedFiles);

      for (Path newPath : newPaths) {
        String newPathString = newPath.toString();
        pendingFiles.add(newPathString);
        processedFiles.add(newPathString);
        localProcessedFileCount.increment();
      }

      lastScanMillis = System.currentTimeMillis();
    }
  }
      public Properties(
          final List<String> sources, final String output, final Set<String> excludes) {
        myRoots = new HashSet<String>(sources);
        final DirectoryScanner.Result result =
            DirectoryScanner.getFiles(myRoots, excludes, ProjectWrapper.this);

        mySources = new HashMap<FileWrapper, FileWrapper>();

        for (FileWrapper fw : result.getFiles()) {
          mySources.put(fw, fw);
        }

        myOutput = output;
        updateOutputStatus();
      }
  @Override
  public Collection<Partition<AbstractFileInputOperator<T>>> definePartitions(
      Collection<Partition<AbstractFileInputOperator<T>>> partitions, PartitioningContext context) {
    lastRepartition = System.currentTimeMillis();

    int totalCount = getNewPartitionCount(partitions, context);

    LOG.debug("Computed new partitions: {}", totalCount);

    if (totalCount == partitions.size()) {
      return partitions;
    }

    AbstractFileInputOperator<T> tempOperator =
        partitions.iterator().next().getPartitionedInstance();

    MutableLong tempGlobalNumberOfRetries = tempOperator.globalNumberOfRetries;
    MutableLong tempGlobalNumberOfFailures = tempOperator.globalNumberOfRetries;

    /*
     * Build collective state from all instances of the operator.
     */
    Set<String> totalProcessedFiles = Sets.newHashSet();
    Set<FailedFile> currentFiles = Sets.newHashSet();
    List<DirectoryScanner> oldscanners = Lists.newLinkedList();
    List<FailedFile> totalFailedFiles = Lists.newLinkedList();
    List<String> totalPendingFiles = Lists.newLinkedList();
    Set<Integer> deletedOperators = Sets.newHashSet();

    for (Partition<AbstractFileInputOperator<T>> partition : partitions) {
      AbstractFileInputOperator<T> oper = partition.getPartitionedInstance();
      totalProcessedFiles.addAll(oper.processedFiles);
      totalFailedFiles.addAll(oper.failedFiles);
      totalPendingFiles.addAll(oper.pendingFiles);
      currentFiles.addAll(unfinishedFiles);
      tempGlobalNumberOfRetries.add(oper.localNumberOfRetries);
      tempGlobalNumberOfFailures.add(oper.localNumberOfFailures);
      if (oper.currentFile != null) {
        currentFiles.add(new FailedFile(oper.currentFile, oper.offset));
      }
      oldscanners.add(oper.getScanner());
      deletedOperators.add(oper.operatorId);
    }

    /*
     * Create partitions of scanners, scanner's partition method will do state
     * transfer for DirectoryScanner objects.
     */
    List<DirectoryScanner> scanners = scanner.partition(totalCount, oldscanners);

    Kryo kryo = new Kryo();
    Collection<Partition<AbstractFileInputOperator<T>>> newPartitions =
        Lists.newArrayListWithExpectedSize(totalCount);
    Collection<IdempotentStorageManager> newManagers =
        Lists.newArrayListWithExpectedSize(totalCount);

    for (int i = 0; i < scanners.size(); i++) {

      // Kryo.copy fails as it attempts to clone transient fields
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      Output loutput = new Output(bos);
      kryo.writeObject(loutput, this);
      loutput.close();
      Input lInput = new Input(bos.toByteArray());
      @SuppressWarnings("unchecked")
      AbstractFileInputOperator<T> oper = kryo.readObject(lInput, this.getClass());
      lInput.close();

      DirectoryScanner scn = scanners.get(i);
      oper.setScanner(scn);

      // Do state transfer for processed files.
      oper.processedFiles.addAll(totalProcessedFiles);
      oper.globalNumberOfFailures = tempGlobalNumberOfRetries;
      oper.localNumberOfFailures.setValue(0);
      oper.globalNumberOfRetries = tempGlobalNumberOfFailures;
      oper.localNumberOfRetries.setValue(0);

      /* redistribute unfinished files properly */
      oper.unfinishedFiles.clear();
      oper.currentFile = null;
      oper.offset = 0;
      Iterator<FailedFile> unfinishedIter = currentFiles.iterator();
      while (unfinishedIter.hasNext()) {
        FailedFile unfinishedFile = unfinishedIter.next();
        if (scn.acceptFile(unfinishedFile.path)) {
          oper.unfinishedFiles.add(unfinishedFile);
          unfinishedIter.remove();
        }
      }

      /* transfer failed files */
      oper.failedFiles.clear();
      Iterator<FailedFile> iter = totalFailedFiles.iterator();
      while (iter.hasNext()) {
        FailedFile ff = iter.next();
        if (scn.acceptFile(ff.path)) {
          oper.failedFiles.add(ff);
          iter.remove();
        }
      }

      /* redistribute pending files properly */
      oper.pendingFiles.clear();
      Iterator<String> pendingFilesIterator = totalPendingFiles.iterator();
      while (pendingFilesIterator.hasNext()) {
        String pathString = pendingFilesIterator.next();
        if (scn.acceptFile(pathString)) {
          oper.pendingFiles.add(pathString);
          pendingFilesIterator.remove();
        }
      }
      newPartitions.add(new DefaultPartition<AbstractFileInputOperator<T>>(oper));
      newManagers.add(oper.idempotentStorageManager);
    }

    idempotentStorageManager.partitioned(newManagers, deletedOperators);
    LOG.info("definePartitions called returning {} partitions", newPartitions.size());
    return newPartitions;
  }
  protected void replay(long windowId) {
    // This operator can partition itself dynamically. When that happens a file can be re-hashed
    // to a different partition than the previous one. In order to handle this, the partition loads
    // all the recovery data for a window and then processes only those files which would be hashed
    // to it in the current run.
    try {
      Map<Integer, Object> recoveryDataPerOperator = idempotentStorageManager.load(windowId);

      for (Object recovery : recoveryDataPerOperator.values()) {
        @SuppressWarnings("unchecked")
        LinkedList<RecoveryEntry> recoveryData = (LinkedList<RecoveryEntry>) recovery;

        for (RecoveryEntry recoveryEntry : recoveryData) {
          if (scanner.acceptFile(recoveryEntry.file)) {
            // The operator may have continued processing the same file in multiple windows.
            // So the recovery states of subsequent windows will have an entry for that file however
            // the offset changes.
            // In this case we continue reading from previously opened stream.
            if (currentFile == null
                || !(currentFile.equals(recoveryEntry.file)
                    && offset == recoveryEntry.startOffset)) {
              if (inputStream != null) {
                closeFile(inputStream);
              }
              processedFiles.add(recoveryEntry.file);
              // removing the file from failed and unfinished queues and pending set
              Iterator<FailedFile> failedFileIterator = failedFiles.iterator();
              while (failedFileIterator.hasNext()) {
                FailedFile ff = failedFileIterator.next();
                if (ff.path.equals(recoveryEntry.file) && ff.offset == recoveryEntry.startOffset) {
                  failedFileIterator.remove();
                  break;
                }
              }

              Iterator<FailedFile> unfinishedFileIterator = unfinishedFiles.iterator();
              while (unfinishedFileIterator.hasNext()) {
                FailedFile ff = unfinishedFileIterator.next();
                if (ff.path.equals(recoveryEntry.file) && ff.offset == recoveryEntry.startOffset) {
                  unfinishedFileIterator.remove();
                  break;
                }
              }
              if (pendingFiles.contains(recoveryEntry.file)) {
                pendingFiles.remove(recoveryEntry.file);
              }
              inputStream =
                  retryFailedFile(new FailedFile(recoveryEntry.file, recoveryEntry.startOffset));
              while (offset < recoveryEntry.endOffset) {
                T line = readEntity();
                offset++;
                emit(line);
              }
            } else {
              while (offset < recoveryEntry.endOffset) {
                T line = readEntity();
                offset++;
                emit(line);
              }
            }
          }
        }
      }
    } catch (IOException e) {
      throw new RuntimeException("replay", e);
    }
  }