@Override
public final void close() {
if (state.equals(ReaderWriterState.OPEN)) {
try {
appender.close();
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to close appender " + appender, e);
}
if (count > 0) {
// commit the temp file
try {
if (!fs.rename(tempPath, finalPath)) {
this.state = ReaderWriterState.ERROR;
throw new DatasetWriterException("Failed to move " + tempPath + " to " + finalPath);
}
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to commit " + finalPath, e);
}
LOG.debug(
"Committed {} for appender {} ({} entities)",
new Object[] {finalPath, appender, count});
} else {
// discard the temp file
try {
if (!fs.delete(tempPath, true)) {
this.state = ReaderWriterState.ERROR;
throw new DatasetWriterException("Failed to delete " + tempPath);
}
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to remove temporary file " + tempPath, e);
}
LOG.debug("Discarded {} ({} entities)", tempPath, count);
}
try {
appender.cleanup();
} catch (IOException e) {
throw new DatasetIOException("Failed to clean up " + appender, e);
}
this.state = ReaderWriterState.CLOSED;
} else if (state.equals(ReaderWriterState.ERROR)) {
this.state = ReaderWriterState.CLOSED;
}
}
@Override
public void write(E entity) {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
reusedKey.reuseFor(entity);
DatasetWriter<E> writer = cachedWriters.getIfPresent(reusedKey);
if (writer == null) {
// avoid checking in every whether the entity belongs in the view by only
// checking when a new writer is created
Preconditions.checkArgument(
view.includes(entity), "View %s does not include entity %s", view, entity);
// get a new key because it is stored in the cache
StorageKey key = StorageKey.copy(reusedKey);
try {
writer = cachedWriters.getUnchecked(key);
} catch (UncheckedExecutionException ex) {
throw new IllegalArgumentException(
"Problem creating view for entity: " + entity, ex.getCause());
}
}
writer.write(entity);
}
@Override
public final void initialize() {
Preconditions.checkState(
state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);
// ensure the directory exists
try {
fs.mkdirs(directory);
} catch (IOException ex) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to create path " + directory, ex);
}
// initialize paths
this.finalPath = new Path(directory, uniqueFilename(descriptor.getFormat()));
this.tempPath = tempFilename(finalPath);
this.appender = newAppender(tempPath);
try {
appender.open();
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to open appender " + appender, e);
}
this.count = 0;
LOG.debug("Opened appender {} for {}", appender, finalPath);
this.state = ReaderWriterState.OPEN;
}
@Override
public void flush() {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
try {
appender.flush();
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetWriterException("Failed to flush appender " + appender);
}
}
@Override
public final void write(E entity) {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
try {
appender.append(entity);
count += 1;
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to append " + entity + " to " + appender, e);
}
}
@Override
public void close() {
if (state.equals(ReaderWriterState.OPEN)) {
logger.debug("Closing all cached writers for view:{}", view);
for (DatasetWriter<E> writer : cachedWriters.asMap().values()) {
logger.debug("Closing partition writer:{}", writer);
writer.close();
}
state = ReaderWriterState.CLOSED;
}
}
@Override
public void open() {
Preconditions.checkState(
state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);
logger.debug("Opening partitioned dataset writer w/strategy:{}", partitionStrategy);
cachedWriters =
CacheBuilder.newBuilder()
.maximumSize(maxWriters)
.removalListener(new DatasetWriterCloser<E>())
.build(new DatasetWriterCacheLoader<E>(view));
state = ReaderWriterState.OPEN;
}
@Override
public final void write(E entity) {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
try {
appender.append(entity);
count += 1;
} catch (RuntimeException e) {
Throwables.propagateIfInstanceOf(e, DatasetRecordException.class);
this.state = ReaderWriterState.ERROR;
throw new DatasetOperationException(e, "Failed to append %s to %s", entity, appender);
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to append " + entity + " to " + appender, e);
}
}
@Override
public final void initialize() {
Preconditions.checkState(
state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);
ValidationException.check(
isSupportedFormat(descriptor), "Not a supported format: %s", descriptor.getFormat());
// ensure the directory exists
try {
fs.mkdirs(directory);
} catch (RuntimeException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetOperationException(e, "Failed to create path %s", directory);
} catch (IOException ex) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to create path " + directory, ex);
}
// initialize paths
try {
this.finalPath = new Path(directory, uniqueFilename(descriptor.getFormat()));
this.tempPath = tempFilename(finalPath);
} catch (RuntimeException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetOperationException(e, "Failed to initialize file paths under %s", directory);
}
try {
this.appender = newAppender(tempPath);
appender.open();
} catch (RuntimeException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetOperationException(e, "Failed to open appender %s", appender);
} catch (IOException e) {
this.state = ReaderWriterState.ERROR;
throw new DatasetIOException("Failed to open appender " + appender, e);
}
this.count = 0;
LOG.info("Opened output appender {} for {}", appender, finalPath);
this.state = ReaderWriterState.OPEN;
}
@Override
public void flush() {
Preconditions.checkState(
state.equals(ReaderWriterState.OPEN), "Attempt to write to a writer in state:%s", state);
logger.debug("Flushing all cached writers for view:{}", view);
/*
* There's a potential for flushing entries that are created by other
* threads while looping through the writers. While normally just wasteful,
* on HDFS, this is particularly bad. We should probably do something about
* this, but it will be difficult as Cache (ideally) uses multiple
* partitions to prevent cached writer contention.
*/
for (DatasetWriter<E> writer : cachedWriters.asMap().values()) {
logger.debug("Flushing partition writer:{}", writer);
writer.flush();
}
}
@Override
public void initialize() {
Preconditions.checkState(
state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);
DatasetDescriptor descriptor = view.getDataset().getDescriptor();
ValidationException.check(
FileSystemWriter.isSupportedFormat(descriptor),
"Not a supported format: %s",
descriptor.getFormat());
LOG.debug("Opening partitioned dataset writer w/strategy:{}", partitionStrategy);
cachedWriters =
CacheBuilder.newBuilder()
.maximumSize(maxWriters)
.removalListener(new DatasetWriterCloser<E>())
.build(createCacheLoader());
state = ReaderWriterState.OPEN;
}
@Override
public boolean isOpen() {
return state.equals(ReaderWriterState.OPEN);
}
