private String putDirectoryBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
Long contentLength = metadata.getContentLength();
if (contentLength != null && contentLength != 0) {
throw new IllegalArgumentException("Directory blob cannot have content: " + blobKey);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
Path outputPath = outputFile.toPath();
if (!outputFile.isDirectory() && !outputFile.mkdirs()) {
throw new IOException("Unable to mkdir: " + outputPath);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(outputPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(DIRECTORY_MD5));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", outputPath);
}
} else {
logger.warn("xattr not supported on %s", blobKey);
}
return base16().lowerCase().encode(DIRECTORY_MD5);
}
@Override
public void removeBlob(final String container, final String blobKey) {
filesystemContainerNameValidator.validate(container);
filesystemBlobKeyValidator.validate(blobKey);
String fileName = buildPathStartingFromBaseDir(container, blobKey);
logger.debug("Deleting blob %s", fileName);
File fileToBeDeleted = new File(fileName);
if (fileToBeDeleted.isDirectory()) {
try {
UserDefinedFileAttributeView view =
getUserDefinedFileAttributeView(fileToBeDeleted.toPath());
if (view != null) {
for (String s : view.list()) {
view.delete(s);
}
}
} catch (IOException e) {
logger.debug("Could not delete attributes from %s: %s", fileToBeDeleted, e);
}
}
try {
delete(fileToBeDeleted);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", fileToBeDeleted, e);
}
// now examine if the key of the blob is a complex key (with a directory structure)
// and eventually remove empty directory
removeDirectoriesTreeOfBlobKey(container, blobKey);
}
/** Read the String representation of filesystem attribute, or return null if not present. */
private static String readStringAttributeIfPresent(
UserDefinedFileAttributeView view, Set<String> attributes, String name) throws IOException {
if (!attributes.contains(name)) {
return null;
}
ByteBuffer buf = ByteBuffer.allocate(view.size(name));
view.read(name, buf);
return new String(buf.array(), StandardCharsets.UTF_8);
}
private boolean buildPathAndChecksIfBlobExists(String... tokens) throws IOException {
String path = buildPathStartingFromBaseDir(tokens);
File file = new File(path);
boolean exists = file.exists() && file.isFile();
if (!exists
&& getDirectoryBlobSuffix(tokens[tokens.length - 1]) != null
&& file.isDirectory()) {
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
exists = view != null && view.list().contains(XATTR_CONTENT_MD5);
}
return exists;
}
static Map<String, ByteBuffer> readUserDefinedFileAttributes(Path file) throws IOException {
UserDefinedFileAttributeView view =
getFileAttributeView(file, UserDefinedFileAttributeView.class);
Map<String, ByteBuffer> result = new HashMap<>();
for (String name : view.list()) {
int size = view.size(name);
ByteBuffer bb = ByteBuffer.allocate(size);
int n = view.read(name, bb);
assertTrue(n == size);
bb.flip();
result.put(name, bb);
}
return result;
}
// "randomize" the file attributes of the given file.
static void randomizeAttributes(Path file) throws IOException {
String os = System.getProperty("os.name");
boolean isWindows = os.startsWith("Windows");
boolean isUnix = os.equals("SunOS") || os.equals("Linux");
boolean isDirectory = isDirectory(file, NOFOLLOW_LINKS);
if (isUnix) {
Set<PosixFilePermission> perms = getPosixFilePermissions(file, NOFOLLOW_LINKS);
PosixFilePermission[] toChange = {
PosixFilePermission.GROUP_READ,
PosixFilePermission.GROUP_WRITE,
PosixFilePermission.GROUP_EXECUTE,
PosixFilePermission.OTHERS_READ,
PosixFilePermission.OTHERS_WRITE,
PosixFilePermission.OTHERS_EXECUTE
};
for (PosixFilePermission perm : toChange) {
if (heads()) {
perms.add(perm);
} else {
perms.remove(perm);
}
}
setPosixFilePermissions(file, perms);
}
if (isWindows) {
DosFileAttributeView view =
getFileAttributeView(file, DosFileAttributeView.class, NOFOLLOW_LINKS);
// only set or unset the hidden attribute
view.setHidden(heads());
}
boolean addUserDefinedFileAttributes =
heads() && getFileStore(file).supportsFileAttributeView("xattr");
// remove this when copying a direcory copies its named streams
if (isWindows && isDirectory) addUserDefinedFileAttributes = false;
if (addUserDefinedFileAttributes) {
UserDefinedFileAttributeView view =
getFileAttributeView(file, UserDefinedFileAttributeView.class);
int n = rand.nextInt(16);
while (n > 0) {
byte[] value = new byte[1 + rand.nextInt(100)];
view.write("user." + Integer.toString(n), ByteBuffer.wrap(value));
n--;
}
}
}
public static long getFileKey(Path filePath) {
if (!Files.exists(filePath)) {
return -1;
}
try {
if (OSDetector.isApple()) {
Xattrj xattrj = getXattrj();
if (xattrj == null) {
return -1;
}
String fileKey = xattrj.readAttribute(filePath.toFile(), "fileKey");
if (fileKey == null) {
return -1;
}
return Long.parseLong(fileKey);
} else {
UserDefinedFileAttributeView userDefinedFileAttributeView =
Files.getFileAttributeView(filePath, UserDefinedFileAttributeView.class);
List<String> list = userDefinedFileAttributeView.list();
if (!list.contains("fileKey")) {
return -1;
}
ByteBuffer byteBuffer = ByteBuffer.allocate(userDefinedFileAttributeView.size("fileKey"));
userDefinedFileAttributeView.read("fileKey", byteBuffer);
CharBuffer charBuffer = _CHARSET.decode((ByteBuffer) byteBuffer.flip());
return Long.parseLong(charBuffer.toString());
}
} catch (Exception e) {
_logger.error(e.getMessage(), e);
return -1;
}
}
public static void writeFileKey(Path filePath, String fileKey) {
if (!OSDetector.isWindows()) {
return;
}
File file = filePath.toFile();
if (!file.canWrite()) {
file.setWritable(true);
}
UserDefinedFileAttributeView userDefinedFileAttributeView =
Files.getFileAttributeView(filePath, UserDefinedFileAttributeView.class);
try {
userDefinedFileAttributeView.write("fileKey", _CHARSET.encode(CharBuffer.wrap(fileKey)));
} catch (Exception e) {
_logger.error(e.getMessage(), e);
}
}
/**
* Removes recursively the directory structure of a complex blob key, only if the directory is
* empty
*
* @param container
* @param blobKey
*/
private void removeDirectoriesTreeOfBlobKey(String container, String blobKey) {
String normalizedBlobKey = denormalize(blobKey);
// exists is no path is present in the blobkey
if (!normalizedBlobKey.contains(File.separator)) return;
File file = new File(normalizedBlobKey);
// TODO
// "/media/data/works/java/amazon/jclouds/master/filesystem/aa/bb/cc/dd/eef6f0c8-0206-460b-8870-352e6019893c.txt"
String parentPath = file.getParent();
// no need to manage "/" parentPath, because "/" cannot be used as start
// char of blobkey
if (!isNullOrEmpty(parentPath)) {
// remove parent directory only it's empty
File directory = new File(buildPathStartingFromBaseDir(container, parentPath));
// don't delete directory if it's a directory blob
try {
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(directory.toPath());
if (view == null) { // OSX HFS+ does not support UserDefinedFileAttributeView
logger.debug("Could not look for attributes from %s", directory);
} else if (!view.list().isEmpty()) {
return;
}
} catch (IOException e) {
logger.debug("Could not look for attributes from %s: %s", directory, e);
}
String[] children = directory.list();
if (null == children || children.length == 0) {
try {
delete(directory);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", directory, e);
return;
}
// recursively call for removing other path
removeDirectoriesTreeOfBlobKey(container, parentPath);
}
}
}
public static String getFileKey(Path filePath) {
if (!Files.exists(filePath)) {
return "";
}
try {
if (OSDetector.isWindows()) {
UserDefinedFileAttributeView userDefinedFileAttributeView =
Files.getFileAttributeView(filePath, UserDefinedFileAttributeView.class);
List<String> list = userDefinedFileAttributeView.list();
if (!list.contains("fileKey")) {
return "";
}
ByteBuffer byteBuffer = ByteBuffer.allocate(userDefinedFileAttributeView.size("fileKey"));
userDefinedFileAttributeView.read("fileKey", byteBuffer);
CharBuffer charBuffer = _CHARSET.decode((ByteBuffer) byteBuffer.flip());
return charBuffer.toString();
} else {
BasicFileAttributes basicFileAttributes =
Files.readAttributes(filePath, BasicFileAttributes.class);
Object fileKey = basicFileAttributes.fileKey();
return fileKey.toString();
}
} catch (Exception e) {
_logger.error(e.getMessage(), e);
return "";
}
}
protected static void doWriteFileKey(Path filePath, String fileKey) {
if (hasFileKey(filePath, Long.parseLong(fileKey))) {
return;
}
if (OSDetector.isApple()) {
Xattrj xattrj = getXattrj();
if (xattrj == null) {
return;
}
File file = filePath.toFile();
if (!file.canWrite()) {
file.setWritable(true);
}
xattrj.writeAttribute(file, "fileKey", fileKey);
} else {
File file = filePath.toFile();
if (!file.canWrite()) {
file.setWritable(true);
}
UserDefinedFileAttributeView userDefinedFileAttributeView =
Files.getFileAttributeView(filePath, UserDefinedFileAttributeView.class);
try {
userDefinedFileAttributeView.write("fileKey", _CHARSET.encode(CharBuffer.wrap(fileKey)));
} catch (Exception e) {
_logger.error(e.getMessage(), e);
}
}
}
private void writeCommonMetadataAttr(UserDefinedFileAttributeView view, Blob blob)
throws IOException {
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
writeStringAttributeIfPresent(view, XATTR_CACHE_CONTROL, metadata.getCacheControl());
writeStringAttributeIfPresent(
view, XATTR_CONTENT_DISPOSITION, metadata.getContentDisposition());
writeStringAttributeIfPresent(view, XATTR_CONTENT_ENCODING, metadata.getContentEncoding());
writeStringAttributeIfPresent(view, XATTR_CONTENT_LANGUAGE, metadata.getContentLanguage());
writeStringAttributeIfPresent(view, XATTR_CONTENT_TYPE, metadata.getContentType());
Date expires = metadata.getExpires();
if (expires != null) {
ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES).putLong(expires.getTime());
buf.flip();
view.write(XATTR_EXPIRES, buf);
}
for (Map.Entry<String, String> entry : blob.getMetadata().getUserMetadata().entrySet()) {
writeStringAttributeIfPresent(
view, XATTR_USER_METADATA_PREFIX + entry.getKey(), entry.getValue());
}
}
/** Write an filesystem attribute, if its value is non-null. */
private static void writeStringAttributeIfPresent(
UserDefinedFileAttributeView view, String name, String value) throws IOException {
if (value != null) {
view.write(name, ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)));
}
}
@Override
public String putBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
Payload payload = blob.getPayload();
filesystemContainerNameValidator.validate(containerName);
filesystemBlobKeyValidator.validate(blobKey);
if (getDirectoryBlobSuffix(blobKey) != null) {
return putDirectoryBlob(containerName, blob);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
// TODO: should we use a known suffix to filter these out during list?
String tmpBlobName = blobKey + "-" + UUID.randomUUID();
File tmpFile = getFileForBlobKey(containerName, tmpBlobName);
Path tmpPath = tmpFile.toPath();
HashingInputStream his = null;
try {
Files.createParentDirs(tmpFile);
his = new HashingInputStream(Hashing.md5(), payload.openStream());
long actualSize = Files.asByteSink(tmpFile).writeFrom(his);
Long expectedSize = blob.getMetadata().getContentMetadata().getContentLength();
if (expectedSize != null && actualSize != expectedSize) {
throw new IOException(
"Content-Length mismatch, actual: " + actualSize + " expected: " + expectedSize);
}
HashCode actualHashCode = his.hash();
HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
throw new IOException(
"MD5 hash code mismatch, actual: " + actualHashCode + " expected: " + expectedHashCode);
}
payload.getContentMetadata().setContentMD5(actualHashCode);
if (outputFile.exists()) {
delete(outputFile);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(tmpPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(actualHashCode.asBytes()));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", tmpPath);
}
}
setBlobAccess(containerName, tmpBlobName, BlobAccess.PRIVATE);
if (!tmpFile.renameTo(outputFile)) {
throw new RuntimeException("Could not rename file " + tmpFile + " to " + outputFile);
}
return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (tmpFile != null) {
try {
delete(tmpFile);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", tmpFile, e);
}
}
throw ex;
} finally {
closeQuietly(his);
if (payload != null) {
payload.release();
}
}
}
@Override
public Blob getBlob(final String container, final String key) {
BlobBuilder builder = blobBuilders.get();
builder.name(key);
File file = getFileForBlobKey(container, key);
ByteSource byteSource;
if (getDirectoryBlobSuffix(key) != null) {
logger.debug("%s - %s is a directory", container, key);
byteSource = ByteSource.empty();
} else {
byteSource = Files.asByteSource(file);
}
try {
String cacheControl = null;
String contentDisposition = null;
String contentEncoding = null;
String contentLanguage = null;
String contentType = null;
HashCode hashCode = null;
Date expires = null;
ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
if (view != null) {
Set<String> attributes = ImmutableSet.copyOf(view.list());
cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
contentDisposition =
readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
if (contentType == null && autoDetectContentType) {
contentType = probeContentType(file.toPath());
}
if (attributes.contains(XATTR_CONTENT_MD5)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
view.read(XATTR_CONTENT_MD5, buf);
hashCode = HashCode.fromBytes(buf.array());
}
if (attributes.contains(XATTR_EXPIRES)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
view.read(XATTR_EXPIRES, buf);
buf.flip();
expires = new Date(buf.asLongBuffer().get());
}
for (String attribute : attributes) {
if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
continue;
}
String value = readStringAttributeIfPresent(view, attributes, attribute);
userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
}
builder
.payload(byteSource)
.cacheControl(cacheControl)
.contentDisposition(contentDisposition)
.contentEncoding(contentEncoding)
.contentLanguage(contentLanguage)
.contentLength(byteSource.size())
.contentMD5(hashCode)
.contentType(contentType)
.expires(expires)
.userMetadata(userMetadata.build());
} else {
builder
.payload(byteSource)
.contentLength(byteSource.size())
.contentMD5(byteSource.hash(Hashing.md5()).asBytes());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(
base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}