Ejemplo n.º 1
0
  @Override
  public final void close() {
    if (state.equals(ReaderWriterState.OPEN)) {
      try {
        appender.close();
      } catch (IOException e) {
        this.state = ReaderWriterState.ERROR;
        throw new DatasetIOException("Failed to close appender " + appender, e);
      }

      if (count > 0) {
        // commit the temp file
        try {
          if (!fs.rename(tempPath, finalPath)) {
            this.state = ReaderWriterState.ERROR;
            throw new DatasetWriterException("Failed to move " + tempPath + " to " + finalPath);
          }
        } catch (IOException e) {
          this.state = ReaderWriterState.ERROR;
          throw new DatasetIOException("Failed to commit " + finalPath, e);
        }

        LOG.debug(
            "Committed {} for appender {} ({} entities)",
            new Object[] {finalPath, appender, count});
      } else {
        // discard the temp file
        try {
          if (!fs.delete(tempPath, true)) {
            this.state = ReaderWriterState.ERROR;
            throw new DatasetWriterException("Failed to delete " + tempPath);
          }
        } catch (IOException e) {
          this.state = ReaderWriterState.ERROR;
          throw new DatasetIOException("Failed to remove temporary file " + tempPath, e);
        }

        LOG.debug("Discarded {} ({} entities)", tempPath, count);
      }

      try {
        appender.cleanup();
      } catch (IOException e) {
        throw new DatasetIOException("Failed to clean up " + appender, e);
      }

      this.state = ReaderWriterState.CLOSED;

    } else if (state.equals(ReaderWriterState.ERROR)) {
      this.state = ReaderWriterState.CLOSED;
    }
  }
Ejemplo n.º 2
0
  @Override
  public void write(E entity) {
    Preconditions.checkState(
        state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);

    reusedKey.reuseFor(entity);

    DatasetWriter<E> writer = cachedWriters.getIfPresent(reusedKey);
    if (writer == null) {
      // avoid checking in every whether the entity belongs in the view by only
      // checking when a new writer is created
      Preconditions.checkArgument(
          view.includes(entity), "View %s does not include entity %s", view, entity);
      // get a new key because it is stored in the cache
      StorageKey key = StorageKey.copy(reusedKey);
      try {
        writer = cachedWriters.getUnchecked(key);
      } catch (UncheckedExecutionException ex) {
        throw new IllegalArgumentException(
            "Problem creating view for entity: " + entity, ex.getCause());
      }
    }

    writer.write(entity);
  }
Ejemplo n.º 3
0
  @Override
  public final void initialize() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);

    // ensure the directory exists
    try {
      fs.mkdirs(directory);
    } catch (IOException ex) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to create path " + directory, ex);
    }

    // initialize paths
    this.finalPath = new Path(directory, uniqueFilename(descriptor.getFormat()));
    this.tempPath = tempFilename(finalPath);
    this.appender = newAppender(tempPath);

    try {
      appender.open();
    } catch (IOException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to open appender " + appender, e);
    }

    this.count = 0;

    LOG.debug("Opened appender {} for {}", appender, finalPath);

    this.state = ReaderWriterState.OPEN;
  }
Ejemplo n.º 4
0
 @Override
 public void flush() {
   Preconditions.checkState(
       state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
   try {
     appender.flush();
   } catch (IOException e) {
     this.state = ReaderWriterState.ERROR;
     throw new DatasetWriterException("Failed to flush appender " + appender);
   }
 }
Ejemplo n.º 5
0
  @Override
  public final void write(E entity) {
    Preconditions.checkState(
        state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);

    try {
      appender.append(entity);
      count += 1;
    } catch (IOException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to append " + entity + " to " + appender, e);
    }
  }
Ejemplo n.º 6
0
  @Override
  public void close() {
    if (state.equals(ReaderWriterState.OPEN)) {

      logger.debug("Closing all cached writers for view:{}", view);

      for (DatasetWriter<E> writer : cachedWriters.asMap().values()) {
        logger.debug("Closing partition writer:{}", writer);
        writer.close();
      }

      state = ReaderWriterState.CLOSED;
    }
  }
Ejemplo n.º 7
0
  @Override
  public void open() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);

    logger.debug("Opening partitioned dataset writer w/strategy:{}", partitionStrategy);

    cachedWriters =
        CacheBuilder.newBuilder()
            .maximumSize(maxWriters)
            .removalListener(new DatasetWriterCloser<E>())
            .build(new DatasetWriterCacheLoader<E>(view));

    state = ReaderWriterState.OPEN;
  }
Ejemplo n.º 8
0
  @Override
  public final void write(E entity) {
    Preconditions.checkState(
        state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);

    try {
      appender.append(entity);
      count += 1;
    } catch (RuntimeException e) {
      Throwables.propagateIfInstanceOf(e, DatasetRecordException.class);
      this.state = ReaderWriterState.ERROR;
      throw new DatasetOperationException(e, "Failed to append %s to %s", entity, appender);
    } catch (IOException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to append " + entity + " to " + appender, e);
    }
  }
Ejemplo n.º 9
0
  @Override
  public final void initialize() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);

    ValidationException.check(
        isSupportedFormat(descriptor), "Not a supported format: %s", descriptor.getFormat());

    // ensure the directory exists
    try {
      fs.mkdirs(directory);
    } catch (RuntimeException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetOperationException(e, "Failed to create path %s", directory);
    } catch (IOException ex) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to create path " + directory, ex);
    }

    // initialize paths
    try {
      this.finalPath = new Path(directory, uniqueFilename(descriptor.getFormat()));
      this.tempPath = tempFilename(finalPath);
    } catch (RuntimeException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetOperationException(e, "Failed to initialize file paths under %s", directory);
    }

    try {
      this.appender = newAppender(tempPath);
      appender.open();
    } catch (RuntimeException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetOperationException(e, "Failed to open appender %s", appender);
    } catch (IOException e) {
      this.state = ReaderWriterState.ERROR;
      throw new DatasetIOException("Failed to open appender " + appender, e);
    }

    this.count = 0;

    LOG.info("Opened output appender {} for {}", appender, finalPath);

    this.state = ReaderWriterState.OPEN;
  }
Ejemplo n.º 10
0
  @Override
  public void flush() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);

    logger.debug("Flushing all cached writers for view:{}", view);

    /*
     * There's a potential for flushing entries that are created by other
     * threads while looping through the writers. While normally just wasteful,
     * on HDFS, this is particularly bad. We should probably do something about
     * this, but it will be difficult as Cache (ideally) uses multiple
     * partitions to prevent cached writer contention.
     */
    for (DatasetWriter<E> writer : cachedWriters.asMap().values()) {
      logger.debug("Flushing partition writer:{}", writer);
      writer.flush();
    }
  }
Ejemplo n.º 11
0
  @Override
  public void initialize() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);

    DatasetDescriptor descriptor = view.getDataset().getDescriptor();
    ValidationException.check(
        FileSystemWriter.isSupportedFormat(descriptor),
        "Not a supported format: %s",
        descriptor.getFormat());

    LOG.debug("Opening partitioned dataset writer w/strategy:{}", partitionStrategy);

    cachedWriters =
        CacheBuilder.newBuilder()
            .maximumSize(maxWriters)
            .removalListener(new DatasetWriterCloser<E>())
            .build(createCacheLoader());

    state = ReaderWriterState.OPEN;
  }
Ejemplo n.º 12
0
 @Override
 public boolean isOpen() {
   return state.equals(ReaderWriterState.OPEN);
 }