@Override public void merge(FileSystemDataset<E> update) { DatasetDescriptor updateDescriptor = update.getDescriptor(); if (!updateDescriptor.getFormat().equals(descriptor.getFormat())) { throw new DatasetRepositoryException( "Cannot merge dataset format " + updateDescriptor.getFormat() + " with format " + descriptor.getFormat()); } if (updateDescriptor.isPartitioned() != descriptor.isPartitioned()) { throw new DatasetRepositoryException( "Cannot merge an unpartitioned dataset with a " + " partitioned one or vice versa."); } else if (updateDescriptor.isPartitioned() && descriptor.isPartitioned() && !updateDescriptor.getPartitionStrategy().equals(descriptor.getPartitionStrategy())) { throw new DatasetRepositoryException( "Cannot merge dataset partition strategy " + updateDescriptor.getPartitionStrategy() + " with " + descriptor.getPartitionStrategy()); } if (!updateDescriptor.getSchema().equals(descriptor.getSchema())) { throw new DatasetRepositoryException( "Cannot merge dataset schema " + updateDescriptor.getFormat() + " with schema " + descriptor.getFormat()); } Set<String> addedPartitions = Sets.newHashSet(); for (Path path : update.pathIterator()) { URI relativePath = update.getDirectory().toUri().relativize(path.toUri()); Path newPath = new Path(directory, new Path(relativePath)); Path newPartitionDirectory = newPath.getParent(); try { if (!fileSystem.exists(newPartitionDirectory)) { fileSystem.mkdirs(newPartitionDirectory); } logger.debug("Renaming {} to {}", path, newPath); boolean renameOk = fileSystem.rename(path, newPath); if (!renameOk) { throw new DatasetException( "Dataset merge failed during rename of " + path + " to " + newPath); } } catch (IOException e) { throw new DatasetIOException("Dataset merge failed", e); } if (descriptor.isPartitioned() && partitionListener != null) { String partition = newPartitionDirectory.toString(); if (!addedPartitions.contains(partition)) { partitionListener.partitionAdded(name, partition); addedPartitions.add(partition); } } } }