@Test
public void testWrite() throws Exception {
String streamString = "testContents";
FileStatus status = fs.getFileStatus(testTempPath);
OwnerAndPermission ownerAndPermission =
new OwnerAndPermission(
status.getOwner(),
status.getGroup(),
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
CopyableFile cf = CopyableFileUtils.getTestCopyableFile(ownerAndPermission);
CopyableDatasetMetadata metadata =
new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source")));
WorkUnitState state = new WorkUnitState();
state.setProp(
ConfigurationKeys.WRITER_STAGING_DIR, new Path(testTempPath, "staging").toString());
state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(testTempPath, "output").toString());
state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5));
CopySource.serializeCopyEntity(state, cf);
CopySource.serializeCopyableDataset(state, metadata);
FileAwareInputStreamDataWriter dataWriter = new FileAwareInputStreamDataWriter(state, 1, 0);
FileAwareInputStream fileAwareInputStream =
new FileAwareInputStream(
cf, StreamUtils.convertStream(IOUtils.toInputStream(streamString)));
dataWriter.write(fileAwareInputStream);
dataWriter.commit();
Path writtenFilePath =
new Path(
new Path(
state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR),
cf.getDatasetAndPartition(metadata).identifier()),
cf.getDestination());
Assert.assertEquals(
IOUtils.toString(new FileInputStream(writtenFilePath.toString())), streamString);
}
@Test
public void testCommit() throws IOException {
String destinationExistingToken = "destination";
String destinationAdditionalTokens = "path";
String fileName = "file";
// Asemble destination paths
Path destination =
new Path(
new Path(new Path("/", destinationExistingToken), destinationAdditionalTokens),
fileName);
Path destinationWithoutLeadingSeparator =
new Path(new Path(destinationExistingToken, destinationAdditionalTokens), fileName);
// Create temp directory
File tmpFile = Files.createTempDir();
tmpFile.deleteOnExit();
Path tmpPath = new Path(tmpFile.getAbsolutePath());
// create origin file
Path originFile = new Path(tmpPath, fileName);
this.fs.createNewFile(originFile);
// create stating dir
Path stagingDir = new Path(tmpPath, "staging");
this.fs.mkdirs(stagingDir);
// create output dir
Path outputDir = new Path(tmpPath, "output");
this.fs.mkdirs(outputDir);
// create copyable file
FileStatus status = this.fs.getFileStatus(originFile);
FsPermission readWrite =
new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE);
FsPermission dirReadWrite =
new FsPermission(FsAction.ALL, FsAction.READ_WRITE, FsAction.READ_WRITE);
OwnerAndPermission ownerAndPermission =
new OwnerAndPermission(status.getOwner(), status.getGroup(), readWrite);
List<OwnerAndPermission> ancestorOwnerAndPermissions = Lists.newArrayList();
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
ancestorOwnerAndPermissions.add(ownerAndPermission);
Properties properties = new Properties();
properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
CopyableFile cf =
CopyableFile.fromOriginAndDestination(
this.fs,
status,
destination,
CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties)
.publishDir(new Path("/target"))
.preserve(PreserveAttributes.fromMnemonicString(""))
.build())
.destinationOwnerAndPermission(ownerAndPermission)
.ancestorsOwnerAndPermission(ancestorOwnerAndPermissions)
.build();
// create work unit state
WorkUnitState state = new WorkUnitState();
state.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir.toUri().getPath());
state.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir.toUri().getPath());
state.setProp(ConfigurationKeys.WRITER_FILE_PATH, RandomStringUtils.randomAlphabetic(5));
CopyableDatasetMetadata metadata =
new CopyableDatasetMetadata(new TestCopyableDataset(new Path("/source")));
CopySource.serializeCopyEntity(state, cf);
CopySource.serializeCopyableDataset(state, metadata);
// create writer
FileAwareInputStreamDataWriter writer = new FileAwareInputStreamDataWriter(state, 1, 0);
// create output of writer.write
Path writtenFile = writer.getStagingFilePath(cf);
this.fs.mkdirs(writtenFile.getParent());
this.fs.createNewFile(writtenFile);
// create existing directories in writer output
Path outputRoot =
FileAwareInputStreamDataWriter.getPartitionOutputRoot(
outputDir, cf.getDatasetAndPartition(metadata));
Path existingOutputPath = new Path(outputRoot, destinationExistingToken);
this.fs.mkdirs(existingOutputPath);
FileStatus fileStatus = this.fs.getFileStatus(existingOutputPath);
FsPermission existingPathPermission = fileStatus.getPermission();
// check initial state of the relevant directories
Assert.assertTrue(this.fs.exists(existingOutputPath));
Assert.assertEquals(this.fs.listStatus(existingOutputPath).length, 0);
writer.actualProcessedCopyableFile = Optional.of(cf);
// commit
writer.commit();
// check state of relevant paths after commit
Path expectedOutputPath = new Path(outputRoot, destinationWithoutLeadingSeparator);
Assert.assertTrue(this.fs.exists(expectedOutputPath));
fileStatus = this.fs.getFileStatus(expectedOutputPath);
Assert.assertEquals(fileStatus.getOwner(), ownerAndPermission.getOwner());
Assert.assertEquals(fileStatus.getGroup(), ownerAndPermission.getGroup());
Assert.assertEquals(fileStatus.getPermission(), readWrite);
// parent should have permissions set correctly
fileStatus = this.fs.getFileStatus(expectedOutputPath.getParent());
Assert.assertEquals(fileStatus.getPermission(), dirReadWrite);
// previously existing paths should not have permissions changed
fileStatus = this.fs.getFileStatus(existingOutputPath);
Assert.assertEquals(fileStatus.getPermission(), existingPathPermission);
Assert.assertFalse(this.fs.exists(writer.stagingDir));
}