public static void main(String[] args) throws Exception {
System.out.println("Starting Cache Client...");
int bucket;
List<CacheServiceInterface> serverlist = new ArrayList<CacheServiceInterface>();
serverlist.add(new DistributedCacheService("http://localhost:3000"));
serverlist.add(new DistributedCacheService("http://localhost:3001"));
serverlist.add(new DistributedCacheService("http://localhost:3002"));
CharSequence val;
for (int key = 1; key <= 10; key++) {
val = generateRandomChar();
bucket =
Hashing.consistentHash(
Hashing.md5().hashString(Integer.toString(key)), serverlist.size());
serverlist.get(bucket).put(key, val.toString());
System.out.println("put (key " + key + " and value " + val + ")" + " in bucket " + bucket);
}
for (int key = 1; key <= 10; key++) {
bucket =
Hashing.consistentHash(
Hashing.md5().hashString(Integer.toString(key)), serverlist.size());
System.out.println(
"get (key "
+ key
+ " and value "
+ serverlist.get(bucket).get(key)
+ ")"
+ "from bucket "
+ bucket);
}
System.out.println("Exiting Cache Client...");
}
public static void main(String[] args) throws Exception {
System.out.println("Starting Cache Client...");
String[] value = {
"0",
"Ferrari 250 GTO",
"Ferrari 250 Testa Rossa",
"Jaguar XJ13",
"Mercedes-Benz SLR McLaren",
"Ferrari 330 P4",
"Maybach Exelero",
"Rolls-Royce Hyperion",
"Lamborghini Veneno",
"Zenvo ST1",
"Audi Le Mans Concept",
" McLaren X-1 Concept",
"Koenigsegg CCXR Trevita"
};
List<DistributedCacheService> server = new ArrayList<DistributedCacheService>();
server.add(new DistributedCacheService("http://localhost:3000"));
server.add(new DistributedCacheService("http://localhost:3001"));
server.add(new DistributedCacheService("http://localhost:3002"));
System.out.println(" ----------------------Putting values to server------------------");
for (int putkey = 1; putkey <= 12; putkey++) {
int bucket =
Hashing.consistentHash(Hashing.md5().hashString(Integer.toString(putkey)), server.size());
server.get(bucket).put(putkey, value[putkey]);
System.out.println(
"The key value pair "
+ putkey
+ "-"
+ value[putkey]
+ " is assigned to server "
+ bucket);
}
System.out.println(" ----------------------Getting values from server------------------");
for (int getkey = 1; getkey <= 12; getkey++) {
int bucket =
Hashing.consistentHash(Hashing.md5().hashString(Integer.toString(getkey)), server.size());
System.out.println(
"The key value pair "
+ getkey
+ "-"
+ server.get(bucket).get(getkey)
+ " is received to server "
+ bucket);
}
System.out.println(" ------------------------------Terminated---------------------------");
}
public static void main(String[] args) throws IOException {
Closer closer = Closer.create();
// copy a file
File origin = new File("join_temp");
File copy = new File("target_temp");
try {
BufferedReader reader = new BufferedReader(new FileReader("join_temp"));
BufferedWriter writer = new BufferedWriter(new FileWriter("target_temp"));
closer.register(reader);
closer.register(writer);
String line;
while ((line = reader.readLine()) != null) {
writer.write(line);
}
} catch (IOException e) {
throw closer.rethrow(e);
} finally {
closer.close();
}
Files.copy(origin, copy);
File moved = new File("moved");
// moving renaming
Files.move(copy, moved);
// working files as string
List<String> lines = Files.readLines(origin, Charsets.UTF_8);
HashCode hashCode = Files.hash(origin, Hashing.md5());
System.out.println(hashCode);
// file write and append
String hamlet = "To be, or not to be it is a question\n";
File write_and_append = new File("write_and_append");
Files.write(hamlet, write_and_append, Charsets.UTF_8);
Files.append(hamlet, write_and_append, Charsets.UTF_8);
// write_and_append.deleteOnExit();
Files.write("OverWrite the file", write_and_append, Charsets.UTF_8);
// ByteSource ByteSink
ByteSource fileBytes = Files.asByteSource(write_and_append);
byte[] readBytes = fileBytes.read();
// equals to pre line -> Files.toByteArray(write_and_append) == readBytes
ByteSink fileByteSink = Files.asByteSink(write_and_append);
fileByteSink.write(Files.toByteArray(write_and_append));
BaseEncoding base64 = BaseEncoding.base64();
System.out.println(base64.encode("123456".getBytes()));
}
@Test
public void testOnConnected() throws Exception {
String password = "******";
when(userService.getPassword()).thenReturn(password);
instance.connect();
botStartedFuture.get(TIMEOUT, TIMEOUT_UNIT);
CountDownLatch latch = new CountDownLatch(1);
doAnswer(
invocation -> {
latch.countDown();
return null;
})
.when(outputIrc)
.joinChannel(DEFAULT_CHANNEL_NAME);
mockTaskService();
instance.connectionStateProperty().set(ConnectionState.CONNECTED);
String md5Password = Hashing.md5().hashString(password, StandardCharsets.UTF_8).toString();
verify(outputIrc).message("NICKSERV", String.format("IDENTIFY %s", md5Password));
assertTrue("Channel has not been joined within timeout", latch.await(TIMEOUT, TIMEOUT_UNIT));
}
public HashCode getOutputHash(FileHashCache fileHashCache) throws IOException {
Hasher hasher = Hashing.md5().newHasher();
for (Path path : getRecordedPaths()) {
hasher.putBytes(fileHashCache.get(path).asBytes());
}
return hasher.hash();
}
/** Walk project references recursively, adding thrift files to the provided list. */
List<File> getRecursiveThriftFiles(MavenProject project, String outputDirectory, List<File> files)
throws IOException {
HashFunction hashFun = Hashing.md5();
if (dependencyIncludes.contains(project.getArtifactId())) {
File dir = new File(new File(project.getFile().getParent(), "target"), outputDirectory);
if (dir.exists()) {
URI baseDir = getFileURI(dir);
for (File f : findThriftFilesInDirectory(dir)) {
URI fileURI = getFileURI(f);
String relPath = baseDir.relativize(fileURI).getPath();
File destFolder = getResourcesOutputDirectory();
destFolder.mkdirs();
File destFile = new File(destFolder, relPath);
if (!destFile.exists()
|| (destFile.isFile()
&& !Files.hash(f, hashFun).equals(Files.hash(destFile, hashFun)))) {
getLog()
.info(
format("copying %s to %s", f.getCanonicalPath(), destFile.getCanonicalPath()));
copyFile(f, destFile);
}
files.add(destFile);
}
}
}
Map<String, MavenProject> refs = project.getProjectReferences();
for (String name : refs.keySet()) {
getRecursiveThriftFiles(refs.get(name), outputDirectory, files);
}
return files;
}
public void sendUpUserMessage() throws IOException {
Properties properties = new Properties();
properties.put("metadata.broker.list", "10.1.3.55:9092,10.1.3.56:9092,10.1.3.59:9092");
properties.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig producerConfig = new ProducerConfig(properties);
kafka.javaapi.producer.Producer<String, String> producer =
new kafka.javaapi.producer.Producer<String, String>(producerConfig);
InputStream in = this.getClass().getResourceAsStream("/upusers.csv");
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line = null;
while ((line = reader.readLine()) != null) {
// System.out.println(line);
ArrayList<String> list =
new ArrayList<String>(Arrays.asList(line.replace(";NULL", "").split(",")));
if (list.size() != 0) {
list.remove(0);
String uid = list.remove(0);
String nline =
Hashing.md5()
.hashString(
uid + System.currentTimeMillis() + new Random().nextLong(), Charsets.UTF_8)
+ ","
+ Joiner.on(",").join(list.toArray()).toString();
// String nline = Joiner.on(",").join(list.toArray()).toString();
KeyedMessage<String, String> message = new KeyedMessage<String, String>(TOPIC, nline);
producer.send(message);
// System.out.println(nline);
// System.out.println(nline);
}
}
}
/** Get a Md5 string which is similar to OS Md5sum */
public static String md5(File file) {
try {
HashCode hc = Files.hash(file, Hashing.md5());
return toHex(hc.asBytes());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
static {
HASH_FUNCTIONS.put("md5", Hashing.md5());
HASH_FUNCTIONS.put("sha1", Hashing.sha1());
HASH_FUNCTIONS.put("sha256", Hashing.sha256());
HASH_FUNCTIONS.put("sha512", Hashing.sha512());
HASH_FUNCTIONS.put("murmur3_32", Hashing.murmur3_32());
HASH_FUNCTIONS.put("murmur3_128", Hashing.murmur3_128());
}
public Crypto(String passphrase) {
byte[] key = Hashing.md5().hashString(passphrase).asBytes();
keySpec = new SecretKeySpec(key, CIPHER_ALGORITHM);
try {
cipher = Cipher.getInstance(CIPHER_TRANSFORMATION, "BC");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Test
public void mappedServersTest() {
Map<Integer, String> servers = new HashMap<>();
servers.put(0, "server0");
servers.put(1, "server1");
HashFunction md5 = Hashing.md5();
List<PartitionEntry> triggers = generateTriggers(3, 1000);
Map<Integer, String> newPartition;
Map<Integer, String> oldPartition;
print("initial - test 2 servers " + servers.toString());
newPartition = new HashMap<>();
for (PartitionEntry trigger : triggers) {
newPartition.put(
trigger.hashCode(),
servers.get(Hashing.consistentHash(md5.hashInt(trigger.hashCode()), 2)));
}
for (int buckets = 3; buckets < 10; buckets++) {
servers.put(buckets - 1, "server" + (buckets - 1));
print("test " + buckets + " servers " + servers.toString());
oldPartition = newPartition;
newPartition = new HashMap<>();
for (PartitionEntry trigger : triggers) {
newPartition.put(
trigger.hashCode(),
servers.get(Hashing.consistentHash(md5.hashInt(trigger.hashCode()), buckets)));
}
int changes = comparePartitions(oldPartition, newPartition);
print(
"Changes from "
+ (buckets - 1)
+ " to "
+ buckets
+ " servers: "
+ changes
+ " of "
+ oldPartition.size());
print("" + (((float) changes / (float) oldPartition.size()) * 100) + " % moved");
print(
"K("
+ oldPartition.size()
+ ")/n("
+ buckets
+ "): "
+ ((float) oldPartition.size() / (float) buckets));
}
}
private void redirectToLoginDialog(final HttpServletResponse res) throws Exception {
final String hash = Hashing.md5().hashString("aWl0cnVzdGl0").toString();
res.sendRedirect(
this.fbOauthUrl
+ "?client_id="
+ this.fbAppId
+ "&redirect_uri="
+ this.fbLoginUrl
+ "&state="
+ hash);
}
/**
* Constants for the Node Layer.
*
* @author Sebastian Graf, University of Konstanz
*/
public interface IConstants {
/** Static to determine key for root node. */
public static final long ROOT_NODE = 0;
/** Static to determine key for null node. */
public static final long NULL_NODE = -1;
/** Hashing function for nodes. */
public static final HashFunction HF = Hashing.md5();
/** Static type key for normal txpes. */
public static final int TYPE_KEY = NamePageHash.generateHashForString("xs:untyped");
}
private boolean haveSameContents(File file, final JarFile jar, final JarEntry entry)
throws IOException {
HashFunction hashFun = Hashing.md5();
HashCode fileHash = Files.hash(file, hashFun);
HashCode streamHash =
ByteStreams.hash(
new InputSupplier<InputStream>() {
public InputStream getInput() throws IOException {
return jar.getInputStream(entry);
}
},
hashFun);
return fileHash.equals(streamHash);
}
private void deserializeRule(DeserializationContext context, Build.Rule rulePb)
throws PackageDeserializationException, InterruptedException {
Location ruleLocation = EmptyLocation.INSTANCE;
RuleClass ruleClass = packageDeserializationEnvironment.getRuleClass(rulePb, ruleLocation);
Map<String, ParsedAttributeValue> attributeValues = new HashMap<>();
AttributesToDeserialize attrToDeserialize =
packageDeserializationEnvironment.attributesToDeserialize();
Hasher hasher = Hashing.md5().newHasher();
for (Build.Attribute attrPb : rulePb.getAttributeList()) {
Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType();
attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb));
if (attrToDeserialize.addSyntheticAttributeHash) {
// TODO(bazel-team): This might give false positives because of explicit vs implicit.
hasher.putBytes(attrPb.toByteArray());
}
}
AttributeContainerWithoutLocation attributeContainer =
new AttributeContainerWithoutLocation(ruleClass, hasher.hash());
Label ruleLabel = deserializeLabel(rulePb.getName());
try {
Rule rule =
createRuleWithParsedAttributeValues(
ruleClass,
ruleLabel,
context.packageBuilder,
ruleLocation,
attributeValues,
NullEventHandler.INSTANCE,
attributeContainer);
context.packageBuilder.addRule(rule);
// Remove the attribute after it is added to package in order to pass the validations
// and be able to compute all the outputs.
if (attrToDeserialize != DESERIALIZE_ALL_ATTRS) {
for (String attrName : attributeValues.keySet()) {
Attribute attribute = ruleClass.getAttributeByName(attrName);
if (!(attrToDeserialize.shouldKeepAttributeWithName.apply(attrName)
|| BuildType.isLabelType(attribute.getType()))) {
attributeContainer.clearIfNotLabel(attrName);
}
}
}
Preconditions.checkState(!rule.containsErrors());
} catch (NameConflictException | LabelSyntaxException e) {
throw new PackageDeserializationException(e);
}
}
public static String getImageUploadIdentifier(String s, long l, int i) {
s =
String.valueOf(
Hashing.md5()
.hashUnencodedChars(
(new StringBuilder(String.valueOf(s).length() + 21))
.append(s)
.append("@")
.append(l)
.toString()));
return (new StringBuilder(String.valueOf(s).length() + 12))
.append(s)
.append("/")
.append(i)
.toString();
}
public static void main(String[] args) throws Exception {
System.out.println("Starting Cache Client...");
ArrayList<CacheServiceInterface> servers = new ArrayList<>();
servers.add(new DistributedCacheService("http://localhost:3000"));
servers.add(new DistributedCacheService("http://localhost:3001"));
servers.add(new DistributedCacheService("http://localhost:3002"));
ConsistentHash hash = new ConsistentHash(Hashing.md5(), 3, servers);
for (long i = 1; i <= 10; i++) {
String aux = Character.toString((char) ('a' + i - 1));
hash.get(i).put(i, aux);
int bucket = servers.indexOf(hash.get(i));
System.out.println(
i + " routed to: " + ((DistributedCacheService) servers.get(bucket)).getUrl());
}
for (long i = 1; i <= 10; i++) {
System.out.println("GET(" + i + ") ==> " + hash.get(i).get(i));
}
System.out.println("Existing Cache Client...");
}
/**
* 创建注册中心.
*
* @param connectString 注册中心连接字符串
* @param namespace 注册中心命名空间
* @param digest 注册中心凭证
* @return 注册中心对象
*/
public static CoordinatorRegistryCenter createCoordinatorRegistryCenter(
final String connectString, final String namespace, final Optional<String> digest) {
Hasher hasher =
Hashing.md5()
.newHasher()
.putString(connectString, Charsets.UTF_8)
.putString(namespace, Charsets.UTF_8);
if (digest.isPresent()) {
hasher.putString(digest.get(), Charsets.UTF_8);
}
HashCode hashCode = hasher.hash();
if (regCenterMap.containsKey(hashCode)) {
return regCenterMap.get(hashCode);
}
ZookeeperConfiguration zkConfig = new ZookeeperConfiguration(connectString, namespace);
if (digest.isPresent()) {
zkConfig.setDigest(digest.get());
}
CoordinatorRegistryCenter result = new ZookeeperRegistryCenter(zkConfig);
result.init();
regCenterMap.putIfAbsent(hashCode, result);
return result;
}
private Statement generateStatement(final String sql, final String dataSourceName)
throws SQLException {
HashCode hashCode =
Hashing.md5()
.newHasher()
.putString(sql, Charsets.UTF_8)
.putString(dataSourceName, Charsets.UTF_8)
.hash();
if (cachedRoutedStatements.containsKey(hashCode)) {
return cachedRoutedStatements.get(hashCode);
}
Connection connection = shardingConnection.getConnection(dataSourceName);
Statement result;
if (0 == resultSetHoldability) {
result = connection.createStatement(resultSetType, resultSetConcurrency);
} else {
result =
connection.createStatement(resultSetType, resultSetConcurrency, resultSetHoldability);
}
replayMethodsInvocation(result);
cachedRoutedStatements.put(hashCode, result);
return result;
}
@Override
public Blob getBlob(final String container, final String key) {
BlobBuilder builder = blobBuilders.get();
builder.name(key);
File file = getFileForBlobKey(container, key);
ByteSource byteSource;
if (getDirectoryBlobSuffix(key) != null) {
logger.debug("%s - %s is a directory", container, key);
byteSource = ByteSource.empty();
} else {
byteSource = Files.asByteSource(file);
}
try {
String cacheControl = null;
String contentDisposition = null;
String contentEncoding = null;
String contentLanguage = null;
String contentType = null;
HashCode hashCode = null;
Date expires = null;
ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
if (view != null) {
Set<String> attributes = ImmutableSet.copyOf(view.list());
cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
contentDisposition =
readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
if (contentType == null && autoDetectContentType) {
contentType = probeContentType(file.toPath());
}
if (attributes.contains(XATTR_CONTENT_MD5)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
view.read(XATTR_CONTENT_MD5, buf);
hashCode = HashCode.fromBytes(buf.array());
}
if (attributes.contains(XATTR_EXPIRES)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
view.read(XATTR_EXPIRES, buf);
buf.flip();
expires = new Date(buf.asLongBuffer().get());
}
for (String attribute : attributes) {
if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
continue;
}
String value = readStringAttributeIfPresent(view, attributes, attribute);
userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
}
builder
.payload(byteSource)
.cacheControl(cacheControl)
.contentDisposition(contentDisposition)
.contentEncoding(contentEncoding)
.contentLanguage(contentLanguage)
.contentLength(byteSource.size())
.contentMD5(hashCode)
.contentType(contentType)
.expires(expires)
.userMetadata(userMetadata.build());
} else {
builder
.payload(byteSource)
.contentLength(byteSource.size())
.contentMD5(byteSource.hash(Hashing.md5()).asBytes());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(
base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
@Override
public String putBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
Payload payload = blob.getPayload();
filesystemContainerNameValidator.validate(containerName);
filesystemBlobKeyValidator.validate(blobKey);
if (getDirectoryBlobSuffix(blobKey) != null) {
return putDirectoryBlob(containerName, blob);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
// TODO: should we use a known suffix to filter these out during list?
String tmpBlobName = blobKey + "-" + UUID.randomUUID();
File tmpFile = getFileForBlobKey(containerName, tmpBlobName);
Path tmpPath = tmpFile.toPath();
HashingInputStream his = null;
try {
Files.createParentDirs(tmpFile);
his = new HashingInputStream(Hashing.md5(), payload.openStream());
long actualSize = Files.asByteSink(tmpFile).writeFrom(his);
Long expectedSize = blob.getMetadata().getContentMetadata().getContentLength();
if (expectedSize != null && actualSize != expectedSize) {
throw new IOException(
"Content-Length mismatch, actual: " + actualSize + " expected: " + expectedSize);
}
HashCode actualHashCode = his.hash();
HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
throw new IOException(
"MD5 hash code mismatch, actual: " + actualHashCode + " expected: " + expectedHashCode);
}
payload.getContentMetadata().setContentMD5(actualHashCode);
if (outputFile.exists()) {
delete(outputFile);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(tmpPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(actualHashCode.asBytes()));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", tmpPath);
}
}
setBlobAccess(containerName, tmpBlobName, BlobAccess.PRIVATE);
if (!tmpFile.renameTo(outputFile)) {
throw new RuntimeException("Could not rename file " + tmpFile + " to " + outputFile);
}
return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (tmpFile != null) {
try {
delete(tmpFile);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", tmpFile, e);
}
}
throw ex;
} finally {
closeQuietly(his);
if (payload != null) {
payload.release();
}
}
}
public class NitroOnDemandLocationGeneratorTest {
private static final String ON_DEMAND_IMI = "on_demand_imi";
private static final int VIDEO_BITRATE = 1500000;
private static final Function<GenreType, String> GENRE_TO_HREF =
new Function<GenreType, String>() {
@Override
public String apply(GenreType input) {
return input.getHref();
}
};
private static final Function<GenreType, String> GENRE_TO_TYPE =
new Function<GenreType, String>() {
@Override
public String apply(GenreType input) {
return input.getType();
}
};
private IdGenerator idGenerator = new NitroIdGenerator(Hashing.md5());
private final OnDemandLocationGenerator generator =
new NitroOnDemandLocationGenerator(idGenerator);
@Test
public void testNonPublisherSpecificFields() {
ItemOnDemandHierarchy onDemandHierarchy = hierarchyFrom(createNitroFilm(true));
ExtendedOnDemandProgramType onDemand =
(ExtendedOnDemandProgramType) generator.generate(onDemandHierarchy, ON_DEMAND_IMI);
assertEquals("P0DT1H30M0.000S", onDemand.getPublishedDuration().toString());
assertEquals("2012-07-03T00:00:00Z", onDemand.getStartOfAvailability().toString());
assertEquals("2013-07-17T00:00:00Z", onDemand.getEndOfAvailability().toString());
assertTrue(onDemand.getFree().isValue());
InstanceDescriptionType instanceDesc = onDemand.getInstanceDescription();
CaptionLanguageType captionLanguage =
Iterables.getOnlyElement(instanceDesc.getCaptionLanguage());
assertTrue(captionLanguage.isClosed());
assertEquals(captionLanguage.getValue(), "en");
List<SignLanguageType> signLanguages = instanceDesc.getSignLanguage();
SignLanguageType signLanguageType = Iterables.getOnlyElement(signLanguages);
assertEquals("bfi", signLanguageType.getValue());
AVAttributesType avAttributes = instanceDesc.getAVAttributes();
AudioAttributesType audioAttrs = Iterables.getOnlyElement(avAttributes.getAudioAttributes());
assertEquals("urn:mpeg:mpeg7:cs:AudioPresentationCS:2001:3", audioAttrs.getMixType().getHref());
VideoAttributesType videoAttrs = avAttributes.getVideoAttributes();
assertEquals(Integer.valueOf(1280), videoAttrs.getHorizontalSize());
assertEquals(Integer.valueOf(720), videoAttrs.getVerticalSize());
assertEquals("16:9", Iterables.getOnlyElement(videoAttrs.getAspectRatio()).getValue());
assertEquals(BigInteger.valueOf(VIDEO_BITRATE), avAttributes.getBitRate().getValue());
assertTrue(avAttributes.getBitRate().isVariable());
UniqueIDType otherId = Iterables.getOnlyElement(instanceDesc.getOtherIdentifier());
assertEquals("b020tm1g", otherId.getValue());
assertEquals("epid.bbc.co.uk", otherId.getAuthority());
}
@Test
public void testNitroSpecificFields() {
Film film = createNitroFilm(false);
ItemOnDemandHierarchy hierarchy = hierarchyFrom(film);
String versionCrid = idGenerator.generateVersionCrid(hierarchy.item(), hierarchy.version());
String onDemandImi =
idGenerator.generateOnDemandImi(
hierarchy.item(), hierarchy.version(), hierarchy.encoding(), hierarchy.location());
ExtendedOnDemandProgramType onDemand =
(ExtendedOnDemandProgramType) generator.generate(hierarchy, onDemandImi);
assertEquals("http://nitro.bbc.co.uk/services/youview", onDemand.getServiceIDRef());
assertEquals(versionCrid, onDemand.getProgram().getCrid());
assertEquals(onDemandImi, onDemand.getInstanceMetadataId());
InstanceDescriptionType instanceDesc = onDemand.getInstanceDescription();
AVAttributesType avAttributes = instanceDesc.getAVAttributes();
List<AudioAttributesType> audioAttributes = avAttributes.getAudioAttributes();
AudioAttributesType audioAttribute = Iterables.getOnlyElement(audioAttributes);
AudioLanguageType audioLanguage = audioAttribute.getAudioLanguage();
assertEquals("urn:tva:metadata:cs:AudioPurposeCS:2007:1", audioLanguage.getPurpose());
assertEquals(true, audioLanguage.isSupplemental());
assertEquals("dubbed", audioLanguage.getType());
}
@Test
public void testGaelicLanguageIsSetForSubtitlesOnAlbaChannel() {
Film film = createAlbaNitroFilm();
ExtendedOnDemandProgramType onDemand = onDemandFor(film);
InstanceDescriptionType instanceDesc = onDemand.getInstanceDescription();
CaptionLanguageType captionLanguage =
Iterables.getOnlyElement(instanceDesc.getCaptionLanguage());
assertTrue(captionLanguage.isClosed());
assertEquals(captionLanguage.getValue(), "gla");
}
@Test
public void testSubtitledFlag() {
CaptionLanguageType captionLanguage =
Iterables.getOnlyElement(
onDemandFor(createNitroFilm(true)).getInstanceDescription().getCaptionLanguage());
assertTrue(captionLanguage.isClosed());
assertEquals("en", captionLanguage.getValue());
assertTrue(
onDemandFor(createNitroFilm(false))
.getInstanceDescription()
.getCaptionLanguage()
.isEmpty());
}
private ExtendedOnDemandProgramType onDemandFor(Item item) {
ItemOnDemandHierarchy hierarchy = hierarchyFrom(item);
String onDemandImi =
idGenerator.generateOnDemandImi(
hierarchy.item(), hierarchy.version(), hierarchy.encoding(), hierarchy.location());
return (ExtendedOnDemandProgramType) generator.generate(hierarchy, onDemandImi);
}
@Test
public void testIfNoActualAvailabilityThenContentNotMarkedAsAvailable() {
ItemOnDemandHierarchy onDemandHierarchy = hierarchyFrom(createNitroFilm(false));
onDemandHierarchy.location().getPolicy().setActualAvailabilityStart(null);
ExtendedOnDemandProgramType onDemand =
(ExtendedOnDemandProgramType) generator.generate(onDemandHierarchy, ON_DEMAND_IMI);
InstanceDescriptionType instanceDesc = onDemand.getInstanceDescription();
Set<String> hrefs =
ImmutableSet.copyOf(Iterables.transform(instanceDesc.getGenre(), GENRE_TO_HREF));
assertTrue(
"No 'media available' genre should be added if no actual availability has been identified",
hrefs.isEmpty());
assertEquals("2012-07-03T00:00:00Z", onDemand.getStartOfAvailability().toString());
assertEquals("2013-07-17T00:00:00Z", onDemand.getEndOfAvailability().toString());
}
@Test
public void testIfActualAvailabilityPresentThenContentMarkedAsAvailable() {
ItemOnDemandHierarchy onDemandHierarchy = hierarchyFrom(createNitroFilm(false));
ExtendedOnDemandProgramType onDemand =
(ExtendedOnDemandProgramType) generator.generate(onDemandHierarchy, ON_DEMAND_IMI);
InstanceDescriptionType instanceDesc = onDemand.getInstanceDescription();
Set<String> hrefs =
ImmutableSet.copyOf(Iterables.transform(instanceDesc.getGenre(), GENRE_TO_HREF));
Set<String> types =
ImmutableSet.copyOf(Iterables.transform(instanceDesc.getGenre(), GENRE_TO_TYPE));
assertEquals(
"http://refdata.youview.com/mpeg7cs/YouViewMediaAvailabilityCS/2010-09-29#media_available",
getOnlyElement(hrefs));
assertEquals("other", getOnlyElement(types));
assertEquals("2012-07-03T00:00:00Z", onDemand.getStartOfAvailability().toString());
assertEquals("2013-07-17T00:00:00Z", onDemand.getEndOfAvailability().toString());
}
private ItemOnDemandHierarchy hierarchyFrom(Item item) {
Version version = Iterables.getOnlyElement(item.getVersions());
Encoding encoding = Iterables.getOnlyElement(version.getManifestedAs());
Location location = Iterables.getOnlyElement(encoding.getAvailableAt());
return new ItemOnDemandHierarchy(item, version, encoding, location);
}
private Film createNitroFilm(boolean subtitled) {
Film film = new Film();
film.setCanonicalUri("http://nitro.bbc.co.uk/programmes/b020tm1g");
film.setPublisher(Publisher.BBC_NITRO);
film.setCountriesOfOrigin(ImmutableSet.of(Countries.GB));
film.setYear(1963);
film.addVersion(createVersion(subtitled));
return film;
}
private Film createAlbaNitroFilm() {
Film film = createNitroFilm(true);
film.setPresentationChannel("http://ref.atlasapi.org/channels/bbcalba");
return film;
}
private Version createVersion(boolean subtitled) {
Version version = new Version();
Restriction restriction = new Restriction();
restriction.setRestricted(true);
version.setManifestedAs(Sets.newHashSet(createEncoding(subtitled)));
version.setDuration(Duration.standardMinutes(90));
version.setCanonicalUri("http://nitro.bbc.co.uk/programmes/b00gszl0");
version.setRestriction(restriction);
return version;
}
private Encoding createEncoding(boolean subtitled) {
Encoding encoding = new Encoding();
encoding.setVideoHorizontalSize(1280);
encoding.setVideoVerticalSize(720);
encoding.setVideoAspectRatio("16:9");
encoding.setVideoBitRate(VIDEO_BITRATE);
encoding.setAudioDescribed(true);
encoding.setSigned(true);
encoding.setSubtitled(subtitled);
encoding.addAvailableAt(createLocation());
return encoding;
}
private Location createLocation() {
Location location = new Location();
Policy policy = new Policy();
policy.setActualAvailabilityStart(new DateTime(2012, 7, 3, 0, 10, 0, DateTimeZone.UTC));
policy.setAvailabilityStart(new DateTime(2012, 7, 3, 0, 0, 0, DateTimeZone.UTC));
policy.setAvailabilityEnd(new DateTime(2013, 7, 17, 0, 0, 0, DateTimeZone.UTC));
location.setPolicy(policy);
return location;
}
}
/**
* Class to monitor config files on local disk. Typical usage is to use the default instance and
* have it monitor as many config files as needed.
*
* <p>The class allows users to specify a watch on a file path and pass in a callback that will be
* invoked whenever an update to the file is detected. Update detection currently works by periodic
* polling; if the last modified time on the file is updated and the content hash has changed, all
* watchers on that file are notified. Note that last modified time is just used as a hint to
* determine whether to check the content and not for versioning.
*
* <p>Objects of this class are thread safe.
*/
public class ConfigFileWatcher {
private static final Logger LOG = LoggerFactory.getLogger(ConfigFileWatcher.class);
private static final HashFunction HASH_FUNCTION = Hashing.md5();
public static final int DEFAULT_POLL_PERIOD_SECONDS = 10;
private static volatile ConfigFileWatcher DEFAULT_INSTANCE = null;
// Thread safety note: only addWatch() can add new entries to this map, and that method
// is synchronized. The reason for using a concurrent map is only to allow the watcher
// thread to concurrently iterate over it.
private final ConcurrentMap<String, ConfigFileInfo> watchedFileMap = Maps.newConcurrentMap();
private final WatcherTask watcherTask;
/** Creates the default ConfigFileWatcher instance on demand. */
public static ConfigFileWatcher defaultInstance() {
if (DEFAULT_INSTANCE == null) {
synchronized (ConfigFileWatcher.class) {
if (DEFAULT_INSTANCE == null) {
DEFAULT_INSTANCE = new ConfigFileWatcher(DEFAULT_POLL_PERIOD_SECONDS);
}
}
}
return DEFAULT_INSTANCE;
}
@VisibleForTesting
ConfigFileWatcher(int pollPeriodSeconds) {
ScheduledExecutorService service =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("ConfigFileWatcher-%d")
.build());
this.watcherTask = new WatcherTask();
service.scheduleWithFixedDelay(
watcherTask, pollPeriodSeconds, pollPeriodSeconds, TimeUnit.SECONDS);
}
/**
* Adds a watch on the specified file. The file must exist, otherwise a FileNotFoundException is
* returned. If the file is deleted after a watch is established, the watcher will log errors but
* continue to monitor it, and resume watching if it is recreated.
*
* @param filePath path to the file to watch.
* @param onUpdate function to call when a change is detected to the file. The entire contents of
* the file will be passed in to the function. Note that onUpdate will be called once before
* this call completes, which facilities initial load of data. This callback is executed
* synchronously on the watcher thread - it is important that the function be non-blocking.
*/
public synchronized void addWatch(String filePath, Function<byte[], Void> onUpdate)
throws IOException {
MorePreconditions.checkNotBlank(filePath);
Preconditions.checkNotNull(onUpdate);
// Read the file and make the initial onUpdate call.
File file = new File(filePath);
ByteSource byteSource = Files.asByteSource(file);
onUpdate.apply(byteSource.read());
// Add the file to our map if it isn't already there, and register the new change watcher.
ConfigFileInfo configFileInfo = watchedFileMap.get(filePath);
if (configFileInfo == null) {
configFileInfo = new ConfigFileInfo(file.lastModified(), byteSource.hash(HASH_FUNCTION));
watchedFileMap.put(filePath, configFileInfo);
}
configFileInfo.changeWatchers.add(onUpdate);
}
@VisibleForTesting
public void runWatcherTaskNow() {
watcherTask.run();
}
/**
* Scheduled task that periodically checks each watched file for updates, and if found to have
* been changed, triggers notifications on all its watchers.
*
* <p>Thread safety note: this task must be run in a single threaded executor; i.e. only one run
* of the task can be active at any time.
*/
private class WatcherTask implements Runnable {
@Override
public void run() {
for (Map.Entry<String, ConfigFileInfo> entry : watchedFileMap.entrySet()) {
String filePath = entry.getKey();
ConfigFileInfo configFileInfo = entry.getValue();
try {
File file = new File(filePath);
long lastModified = file.lastModified();
Preconditions.checkArgument(lastModified > 0L);
if (lastModified != configFileInfo.lastModifiedTimestampMillis) {
configFileInfo.lastModifiedTimestampMillis = lastModified;
ByteSource byteSource = Files.asByteSource(file);
HashCode newContentHash = byteSource.hash(HASH_FUNCTION);
if (!newContentHash.equals(configFileInfo.contentHash)) {
configFileInfo.contentHash = newContentHash;
LOG.info("File {} was modified at {}, notifying watchers.", filePath, lastModified);
byte[] newContents = byteSource.read();
for (Function<byte[], Void> watchers : configFileInfo.changeWatchers) {
try {
watchers.apply(newContents);
} catch (Exception e) {
LOG.error(
"Exception in watcher callback for {}, ignoring. New file contents were: {}",
filePath,
new String(newContents, Charsets.UTF_8),
e);
}
}
} else {
LOG.info(
"File {} was modified at {} but content hash is unchanged.",
filePath,
lastModified);
}
} else {
LOG.debug("File {} not modified since {}", filePath, lastModified);
}
} catch (Exception e) {
// We catch and log exceptions related to the update of any specific file, but
// move on so others aren't affected. Issues can happen for example if the watcher
// races with an external file replace operation; in that case, the next run should
// pick up the update.
// TODO: Consider adding a metric to track this so we can alert on failures.
LOG.error("Config update check failed for {}", filePath, e);
}
}
}
}
/**
* Encapsulates state related to each watched config file.
*
* <p>Thread safety note: 1. changeWatchers is thread safe since it uses a copy-on-write array
* list. 2. lastModifiedTimestampMillis and contentHash aren't safe to update across threads. We
* initialize in addWatch() at construction time, and thereafter only the watcher task thread
* accesses this state, so we are good.
*/
private static class ConfigFileInfo {
private final List<Function<byte[], Void>> changeWatchers = Lists.newCopyOnWriteArrayList();
private long lastModifiedTimestampMillis;
private HashCode contentHash;
public ConfigFileInfo(long lastModifiedTimestampMillis, HashCode contentHash) {
this.lastModifiedTimestampMillis = lastModifiedTimestampMillis;
Preconditions.checkArgument(lastModifiedTimestampMillis > 0L);
this.contentHash = Preconditions.checkNotNull(contentHash);
}
}
}
/**
* Get the MD5 hash of the given stream.
*
* @param fis the input stream to use
* @return a byte array of the MD5 hash
* @throws java.security.NoSuchAlgorithmException if MD5 is not available
* @throws IOException if an I/O error occurs
*/
private static byte[] md5(InputStream fis) throws NoSuchAlgorithmException, IOException {
return ByteStreams.hash(
ByteStreams.newInputStreamSupplier(ByteStreams.toByteArray(fis)), Hashing.md5())
.asBytes();
}
/**
* Get the MD5 hash of the given file.
*
* @param file file to compute a hash on
* @return a byte array of the MD5 hash
* @throws IOException if file cannot be found
* @throws NoSuchAlgorithmException if MD5 is not available
*/
private static byte[] md5(File file) throws NoSuchAlgorithmException, IOException {
return Files.hash(file, Hashing.md5()).asBytes();
}
private static String getUniqueUserFilePrefix(String s) {
return String.valueOf(Hashing.md5().hashUnencodedChars(s).toString()).concat("_");
}
public static String getMd5(String value) {
return Hashing.md5().newHasher().putString(value, Charsets.UTF_8).hash().toString();
}
public void testToString() {
assertEquals("Hashing.md5()", Hashing.md5().toString());
assertEquals("Hashing.sha1()", Hashing.sha1().toString());
assertEquals("Hashing.sha256()", Hashing.sha256().toString());
assertEquals("Hashing.sha512()", Hashing.sha512().toString());
}
/**
* Tests for the MessageDigestHashFunction.
*
* @author Kurt Alfred Kluever
*/
public class MessageDigestHashFunctionTest extends TestCase {
private static final ImmutableSet<String> INPUTS = ImmutableSet.of("", "Z", "foobar");
// From "How Provider Implementations Are Requested and Supplied" from
// http://docs.oracle.com/javase/6/docs/technotes/guides/security/crypto/CryptoSpec.html
// - Some providers may choose to also include alias names.
// - For example, the "SHA-1" algorithm might be referred to as "SHA1".
// - The algorithm name is not case-sensitive.
private static final ImmutableMap<String, HashFunction> ALGORITHMS =
new ImmutableMap.Builder<String, HashFunction>()
.put("MD5", Hashing.md5())
.put("SHA", Hashing.sha1()) // Not the official name, but still works
.put("SHA1", Hashing.sha1()) // Not the official name, but still works
.put("sHa-1", Hashing.sha1()) // Not the official name, but still works
.put("SHA-1", Hashing.sha1())
.put("SHA-256", Hashing.sha256())
.put("SHA-512", Hashing.sha512())
.build();
public void testHashing() {
for (String stringToTest : INPUTS) {
for (String algorithmToTest : ALGORITHMS.keySet()) {
assertMessageDigestHashing(HashTestUtils.ascii(stringToTest), algorithmToTest);
}
}
}
public void testPutAfterHash() {
Hasher sha1 = Hashing.sha1().newHasher();
assertEquals(
"2fd4e1c67a2d28fced849ee1bb76e7391b93eb12",
sha1.putString("The quick brown fox jumps over the lazy dog", Charsets.UTF_8)
.hash()
.toString());
try {
sha1.putInt(42);
fail();
} catch (IllegalStateException expected) {
}
}
public void testHashTwice() {
Hasher sha1 = Hashing.sha1().newHasher();
assertEquals(
"2fd4e1c67a2d28fced849ee1bb76e7391b93eb12",
sha1.putString("The quick brown fox jumps over the lazy dog", Charsets.UTF_8)
.hash()
.toString());
try {
sha1.hash();
fail();
} catch (IllegalStateException expected) {
}
}
public void testToString() {
assertEquals("Hashing.md5()", Hashing.md5().toString());
assertEquals("Hashing.sha1()", Hashing.sha1().toString());
assertEquals("Hashing.sha256()", Hashing.sha256().toString());
assertEquals("Hashing.sha512()", Hashing.sha512().toString());
}
private static void assertMessageDigestHashing(byte[] input, String algorithmName) {
try {
MessageDigest digest = MessageDigest.getInstance(algorithmName);
assertEquals(
HashCode.fromBytes(digest.digest(input)), ALGORITHMS.get(algorithmName).hashBytes(input));
for (int bytes = 4; bytes <= digest.getDigestLength(); bytes++) {
assertEquals(
HashCode.fromBytes(Arrays.copyOf(digest.digest(input), bytes)),
new MessageDigestHashFunction(algorithmName, bytes, algorithmName).hashBytes(input));
}
try {
int maxSize = digest.getDigestLength();
new MessageDigestHashFunction(algorithmName, maxSize + 1, algorithmName);
fail();
} catch (IllegalArgumentException expected) {
}
} catch (NoSuchAlgorithmException nsae) {
throw new AssertionError(nsae);
}
}
}
/**
* FilesystemStorageStrategyImpl implements a blob store that stores objects on the file system.
* Content metadata and user attributes are stored in extended attributes if the file system
* supports them. Directory blobs (blobs that end with a /) cannot have content, but otherwise
* appear in LIST like normal blobs.
*/
public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
private static final String XATTR_CACHE_CONTROL = "user.cache-control";
private static final String XATTR_CONTENT_DISPOSITION = "user.content-disposition";
private static final String XATTR_CONTENT_ENCODING = "user.content-encoding";
private static final String XATTR_CONTENT_LANGUAGE = "user.content-language";
private static final String XATTR_CONTENT_MD5 = "user.content-md5";
private static final String XATTR_CONTENT_TYPE = "user.content-type";
private static final String XATTR_EXPIRES = "user.expires";
private static final String XATTR_USER_METADATA_PREFIX = "user.user-metadata.";
private static final byte[] DIRECTORY_MD5 = Hashing.md5().hashBytes(new byte[0]).asBytes();
private static final String BACK_SLASH = "\\";
@Resource protected Logger logger = Logger.NULL;
protected final Provider<BlobBuilder> blobBuilders;
protected final String baseDirectory;
protected final boolean autoDetectContentType;
protected final FilesystemContainerNameValidator filesystemContainerNameValidator;
protected final FilesystemBlobKeyValidator filesystemBlobKeyValidator;
private final Supplier<Location> defaultLocation;
@Inject
protected FilesystemStorageStrategyImpl(
Provider<BlobBuilder> blobBuilders,
@Named(FilesystemConstants.PROPERTY_BASEDIR) String baseDir,
@Named(FilesystemConstants.PROPERTY_AUTO_DETECT_CONTENT_TYPE) boolean autoDetectContentType,
FilesystemContainerNameValidator filesystemContainerNameValidator,
FilesystemBlobKeyValidator filesystemBlobKeyValidator,
Supplier<Location> defaultLocation) {
this.blobBuilders = checkNotNull(blobBuilders, "filesystem storage strategy blobBuilders");
this.baseDirectory = checkNotNull(baseDir, "filesystem storage strategy base directory");
this.autoDetectContentType = autoDetectContentType;
this.filesystemContainerNameValidator =
checkNotNull(filesystemContainerNameValidator, "filesystem container name validator");
this.filesystemBlobKeyValidator =
checkNotNull(filesystemBlobKeyValidator, "filesystem blob key validator");
this.defaultLocation = defaultLocation;
}
@Override
public boolean containerExists(String container) {
filesystemContainerNameValidator.validate(container);
return directoryExists(container, null);
}
@Override
public Collection<String> getAllContainerNames() {
File[] files = new File(buildPathStartingFromBaseDir()).listFiles();
if (files == null) {
return ImmutableList.of();
}
ImmutableList.Builder<String> containers = ImmutableList.builder();
for (File file : files) {
if (file.isDirectory()) {
containers.add(file.getName());
}
}
return containers.build();
}
@Override
public boolean createContainerInLocation(
String container, Location location, CreateContainerOptions options) {
// TODO: implement location
logger.debug("Creating container %s", container);
filesystemContainerNameValidator.validate(container);
boolean created = createDirectoryWithResult(container, null);
if (created) {
setContainerAccess(
container,
options.isPublicRead() ? ContainerAccess.PUBLIC_READ : ContainerAccess.PRIVATE);
}
return created;
}
@Override
public ContainerAccess getContainerAccess(String container) {
Path path = new File(buildPathStartingFromBaseDir(container)).toPath();
if (isWindows()) {
try {
if (isPrivate(path)) {
return ContainerAccess.PRIVATE;
} else {
return ContainerAccess.PUBLIC_READ;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
Set<PosixFilePermission> permissions;
try {
permissions = getPosixFilePermissions(path);
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
return permissions.contains(PosixFilePermission.OTHERS_READ)
? ContainerAccess.PUBLIC_READ
: ContainerAccess.PRIVATE;
}
}
@Override
public void setContainerAccess(String container, ContainerAccess access) {
Path path = new File(buildPathStartingFromBaseDir(container)).toPath();
if (isWindows()) {
try {
if (access == ContainerAccess.PRIVATE) {
setPrivate(path);
} else {
setPublic(path);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
Set<PosixFilePermission> permissions;
try {
permissions = getPosixFilePermissions(path);
if (access == ContainerAccess.PRIVATE) {
permissions.remove(PosixFilePermission.OTHERS_READ);
} else if (access == ContainerAccess.PUBLIC_READ) {
permissions.add(PosixFilePermission.OTHERS_READ);
}
setPosixFilePermissions(path, permissions);
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
}
}
@Override
public void deleteContainer(String container) {
filesystemContainerNameValidator.validate(container);
if (!containerExists(container)) {
return;
}
deleteDirectory(container, null);
}
@Override
public void clearContainer(final String container) {
clearContainer(container, ListContainerOptions.Builder.recursive());
}
@Override
public void clearContainer(String container, ListContainerOptions options) {
filesystemContainerNameValidator.validate(container);
if (options.getDir() != null) {
container += denormalize("/" + options.getDir());
}
try {
File containerFile = openFolder(container);
File[] children = containerFile.listFiles();
if (null != children) {
for (File child : children)
if (options.isRecursive() || child.isFile()) {
Utils.deleteRecursively(child);
}
}
} catch (IOException e) {
logger.error(e, "An error occurred while clearing container %s", container);
Throwables.propagate(e);
}
}
@Override
public StorageMetadata getContainerMetadata(String container) {
MutableStorageMetadata metadata = new MutableStorageMetadataImpl();
metadata.setName(container);
metadata.setType(StorageType.CONTAINER);
metadata.setLocation(getLocation(container));
Path path = new File(buildPathStartingFromBaseDir(container)).toPath();
BasicFileAttributes attr;
try {
attr = readAttributes(path, BasicFileAttributes.class);
} catch (IOException e) {
throw Throwables.propagate(e);
}
metadata.setCreationDate(new Date(attr.creationTime().toMillis()));
return metadata;
}
@Override
public boolean blobExists(String container, String key) {
filesystemContainerNameValidator.validate(container);
filesystemBlobKeyValidator.validate(key);
try {
return buildPathAndChecksIfBlobExists(container, key);
} catch (IOException e) {
logger.error(e, "An error occurred while checking key %s in container %s", container, key);
throw Throwables.propagate(e);
}
}
/**
* Returns all the blobs key inside a container
*
* @param container
* @return
* @throws IOException
*/
@Override
public Iterable<String> getBlobKeysInsideContainer(String container) throws IOException {
filesystemContainerNameValidator.validate(container);
// check if container exists
// TODO maybe an error is more appropriate
Set<String> blobNames = Sets.newHashSet();
if (!containerExists(container)) {
return blobNames;
}
File containerFile = openFolder(container);
final int containerPathLength = containerFile.getAbsolutePath().length() + 1;
populateBlobKeysInContainer(
containerFile,
blobNames,
new Function<String, String>() {
@Override
public String apply(String string) {
return string.substring(containerPathLength);
}
});
return blobNames;
}
@Override
public Blob getBlob(final String container, final String key) {
BlobBuilder builder = blobBuilders.get();
builder.name(key);
File file = getFileForBlobKey(container, key);
ByteSource byteSource;
if (getDirectoryBlobSuffix(key) != null) {
logger.debug("%s - %s is a directory", container, key);
byteSource = ByteSource.empty();
} else {
byteSource = Files.asByteSource(file);
}
try {
String cacheControl = null;
String contentDisposition = null;
String contentEncoding = null;
String contentLanguage = null;
String contentType = null;
HashCode hashCode = null;
Date expires = null;
ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
if (view != null) {
Set<String> attributes = ImmutableSet.copyOf(view.list());
cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
contentDisposition =
readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
if (contentType == null && autoDetectContentType) {
contentType = probeContentType(file.toPath());
}
if (attributes.contains(XATTR_CONTENT_MD5)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
view.read(XATTR_CONTENT_MD5, buf);
hashCode = HashCode.fromBytes(buf.array());
}
if (attributes.contains(XATTR_EXPIRES)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
view.read(XATTR_EXPIRES, buf);
buf.flip();
expires = new Date(buf.asLongBuffer().get());
}
for (String attribute : attributes) {
if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
continue;
}
String value = readStringAttributeIfPresent(view, attributes, attribute);
userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
}
builder
.payload(byteSource)
.cacheControl(cacheControl)
.contentDisposition(contentDisposition)
.contentEncoding(contentEncoding)
.contentLanguage(contentLanguage)
.contentLength(byteSource.size())
.contentMD5(hashCode)
.contentType(contentType)
.expires(expires)
.userMetadata(userMetadata.build());
} else {
builder
.payload(byteSource)
.contentLength(byteSource.size())
.contentMD5(byteSource.hash(Hashing.md5()).asBytes());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(
base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
private void writeCommonMetadataAttr(UserDefinedFileAttributeView view, Blob blob)
throws IOException {
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
writeStringAttributeIfPresent(view, XATTR_CACHE_CONTROL, metadata.getCacheControl());
writeStringAttributeIfPresent(
view, XATTR_CONTENT_DISPOSITION, metadata.getContentDisposition());
writeStringAttributeIfPresent(view, XATTR_CONTENT_ENCODING, metadata.getContentEncoding());
writeStringAttributeIfPresent(view, XATTR_CONTENT_LANGUAGE, metadata.getContentLanguage());
writeStringAttributeIfPresent(view, XATTR_CONTENT_TYPE, metadata.getContentType());
Date expires = metadata.getExpires();
if (expires != null) {
ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES).putLong(expires.getTime());
buf.flip();
view.write(XATTR_EXPIRES, buf);
}
for (Map.Entry<String, String> entry : blob.getMetadata().getUserMetadata().entrySet()) {
writeStringAttributeIfPresent(
view, XATTR_USER_METADATA_PREFIX + entry.getKey(), entry.getValue());
}
}
private String putDirectoryBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
Long contentLength = metadata.getContentLength();
if (contentLength != null && contentLength != 0) {
throw new IllegalArgumentException("Directory blob cannot have content: " + blobKey);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
Path outputPath = outputFile.toPath();
if (!outputFile.isDirectory() && !outputFile.mkdirs()) {
throw new IOException("Unable to mkdir: " + outputPath);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(outputPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(DIRECTORY_MD5));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", outputPath);
}
} else {
logger.warn("xattr not supported on %s", blobKey);
}
return base16().lowerCase().encode(DIRECTORY_MD5);
}
@Override
public String putBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
Payload payload = blob.getPayload();
filesystemContainerNameValidator.validate(containerName);
filesystemBlobKeyValidator.validate(blobKey);
if (getDirectoryBlobSuffix(blobKey) != null) {
return putDirectoryBlob(containerName, blob);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
// TODO: should we use a known suffix to filter these out during list?
String tmpBlobName = blobKey + "-" + UUID.randomUUID();
File tmpFile = getFileForBlobKey(containerName, tmpBlobName);
Path tmpPath = tmpFile.toPath();
HashingInputStream his = null;
try {
Files.createParentDirs(tmpFile);
his = new HashingInputStream(Hashing.md5(), payload.openStream());
long actualSize = Files.asByteSink(tmpFile).writeFrom(his);
Long expectedSize = blob.getMetadata().getContentMetadata().getContentLength();
if (expectedSize != null && actualSize != expectedSize) {
throw new IOException(
"Content-Length mismatch, actual: " + actualSize + " expected: " + expectedSize);
}
HashCode actualHashCode = his.hash();
HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
throw new IOException(
"MD5 hash code mismatch, actual: " + actualHashCode + " expected: " + expectedHashCode);
}
payload.getContentMetadata().setContentMD5(actualHashCode);
if (outputFile.exists()) {
delete(outputFile);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(tmpPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(actualHashCode.asBytes()));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", tmpPath);
}
}
setBlobAccess(containerName, tmpBlobName, BlobAccess.PRIVATE);
if (!tmpFile.renameTo(outputFile)) {
throw new RuntimeException("Could not rename file " + tmpFile + " to " + outputFile);
}
return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (tmpFile != null) {
try {
delete(tmpFile);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", tmpFile, e);
}
}
throw ex;
} finally {
closeQuietly(his);
if (payload != null) {
payload.release();
}
}
}
@Override
public void removeBlob(final String container, final String blobKey) {
filesystemContainerNameValidator.validate(container);
filesystemBlobKeyValidator.validate(blobKey);
String fileName = buildPathStartingFromBaseDir(container, blobKey);
logger.debug("Deleting blob %s", fileName);
File fileToBeDeleted = new File(fileName);
if (fileToBeDeleted.isDirectory()) {
try {
UserDefinedFileAttributeView view =
getUserDefinedFileAttributeView(fileToBeDeleted.toPath());
if (view != null) {
for (String s : view.list()) {
view.delete(s);
}
}
} catch (IOException e) {
logger.debug("Could not delete attributes from %s: %s", fileToBeDeleted, e);
}
}
try {
delete(fileToBeDeleted);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", fileToBeDeleted, e);
}
// now examine if the key of the blob is a complex key (with a directory structure)
// and eventually remove empty directory
removeDirectoriesTreeOfBlobKey(container, blobKey);
}
@Override
public BlobAccess getBlobAccess(String containerName, String blobName) {
Path path = new File(buildPathStartingFromBaseDir(containerName, blobName)).toPath();
if (isWindows()) {
try {
if (isPrivate(path)) {
return BlobAccess.PRIVATE;
} else {
return BlobAccess.PUBLIC_READ;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
Set<PosixFilePermission> permissions;
try {
permissions = getPosixFilePermissions(path);
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
return permissions.contains(PosixFilePermission.OTHERS_READ)
? BlobAccess.PUBLIC_READ
: BlobAccess.PRIVATE;
}
}
@Override
public void setBlobAccess(String container, String name, BlobAccess access) {
Path path = new File(buildPathStartingFromBaseDir(container, name)).toPath();
if (isWindows()) {
try {
if (access == BlobAccess.PRIVATE) {
setPrivate(path);
} else {
setPublic(path);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
Set<PosixFilePermission> permissions;
try {
permissions = getPosixFilePermissions(path);
if (access == BlobAccess.PRIVATE) {
permissions.remove(PosixFilePermission.OTHERS_READ);
} else if (access == BlobAccess.PUBLIC_READ) {
permissions.add(PosixFilePermission.OTHERS_READ);
}
setPosixFilePermissions(path, permissions);
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
}
}
@Override
public Location getLocation(final String containerName) {
return defaultLocation.get();
}
@Override
public String getSeparator() {
return File.separator;
}
public boolean createContainer(String container) {
filesystemContainerNameValidator.validate(container);
return createContainerInLocation(container, null, CreateContainerOptions.NONE);
}
public Blob newBlob(@ParamValidators({FilesystemBlobKeyValidator.class}) String name) {
filesystemBlobKeyValidator.validate(name);
return blobBuilders.get().name(name).build();
}
/**
* Returns a {@link File} object that links to the blob
*
* @param container
* @param blobKey
* @return
*/
public File getFileForBlobKey(String container, String blobKey) {
filesystemContainerNameValidator.validate(container);
filesystemBlobKeyValidator.validate(blobKey);
String fileName = buildPathStartingFromBaseDir(container, blobKey);
File blobFile = new File(fileName);
return blobFile;
}
public boolean directoryExists(String container, String directory) {
return buildPathAndChecksIfDirectoryExists(container, directory);
}
public void createDirectory(String container, String directory) {
createDirectoryWithResult(container, directory);
}
public void deleteDirectory(String container, String directory) {
// create complete dir path
String fullDirPath = buildPathStartingFromBaseDir(container, directory);
try {
Utils.deleteRecursively(new File(fullDirPath));
} catch (IOException ex) {
logger.error("An error occurred removing directory %s.", fullDirPath);
Throwables.propagate(ex);
}
}
public long countBlobs(String container, ListContainerOptions options) {
// TODO: honor options
try {
return Iterables.size(getBlobKeysInsideContainer(container));
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
}
// ---------------------------------------------------------- Private methods
private boolean buildPathAndChecksIfBlobExists(String... tokens) throws IOException {
String path = buildPathStartingFromBaseDir(tokens);
File file = new File(path);
boolean exists = file.exists() && file.isFile();
if (!exists
&& getDirectoryBlobSuffix(tokens[tokens.length - 1]) != null
&& file.isDirectory()) {
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
exists = view != null && view.list().contains(XATTR_CONTENT_MD5);
}
return exists;
}
private static String getDirectoryBlobSuffix(String key) {
for (String suffix : BlobStoreConstants.DIRECTORY_SUFFIXES) {
if (key.endsWith(suffix)) {
return suffix;
}
}
return null;
}
private static String directoryBlobName(String key) {
String suffix = getDirectoryBlobSuffix(key);
if (suffix != null) {
if (!BlobStoreConstants.DIRECTORY_BLOB_SUFFIX.equals(suffix)) {
key = key.substring(0, key.lastIndexOf(suffix));
}
return key + BlobStoreConstants.DIRECTORY_BLOB_SUFFIX;
}
return null;
}
private UserDefinedFileAttributeView getUserDefinedFileAttributeView(Path path)
throws IOException {
return getFileAttributeView(path, UserDefinedFileAttributeView.class);
}
/**
* Check if the file system resource whose name is obtained applying buildPath on the input path
* tokens is a directory, otherwise a RuntimeException is thrown
*
* @param tokens the tokens that make up the name of the resource on the file system
*/
private boolean buildPathAndChecksIfDirectoryExists(String... tokens) {
String path = buildPathStartingFromBaseDir(tokens);
File file = new File(path);
boolean exists = file.exists() || file.isDirectory();
return exists;
}
/**
* Facility method used to concatenate path tokens normalizing separators
*
* @param pathTokens all the string in the proper order that must be concatenated in order to
* obtain the filename
* @return the resulting string
*/
protected String buildPathStartingFromBaseDir(String... pathTokens) {
String normalizedToken = removeFileSeparatorFromBorders(normalize(baseDirectory), true);
StringBuilder completePath = new StringBuilder(normalizedToken);
if (pathTokens != null && pathTokens.length > 0) {
for (int i = 0; i < pathTokens.length; i++) {
if (pathTokens[i] != null) {
normalizedToken = removeFileSeparatorFromBorders(normalize(pathTokens[i]), false);
completePath.append(File.separator).append(normalizedToken);
}
}
}
return completePath.toString();
}
/**
* Substitutes all the file separator occurrences in the path with a file separator for the
* current operative system
*
* @param pathToBeNormalized
* @return
*/
private static String normalize(String pathToBeNormalized) {
if (null != pathToBeNormalized && pathToBeNormalized.contains(BACK_SLASH)) {
if (!BACK_SLASH.equals(File.separator)) {
return pathToBeNormalized.replace(BACK_SLASH, File.separator);
}
}
return pathToBeNormalized;
}
private static String denormalize(String pathToDenormalize) {
if (null != pathToDenormalize && pathToDenormalize.contains("/")) {
if (BACK_SLASH.equals(File.separator)) {
return pathToDenormalize.replace("/", BACK_SLASH);
}
}
return pathToDenormalize;
}
/**
* Remove leading and trailing separator character from the string.
*
* @param pathToBeCleaned
* @param onlyTrailing only trailing separator char from path
* @return
*/
private String removeFileSeparatorFromBorders(String pathToBeCleaned, boolean onlyTrailing) {
if (null == pathToBeCleaned || pathToBeCleaned.equals("")) return pathToBeCleaned;
int beginIndex = 0;
int endIndex = pathToBeCleaned.length();
// search for separator chars
if (!onlyTrailing) {
if (pathToBeCleaned.substring(0, 1).equals(File.separator)) beginIndex = 1;
}
if (pathToBeCleaned.substring(pathToBeCleaned.length() - 1).equals(File.separator)) endIndex--;
return pathToBeCleaned.substring(beginIndex, endIndex);
}
/**
* Removes recursively the directory structure of a complex blob key, only if the directory is
* empty
*
* @param container
* @param blobKey
*/
private void removeDirectoriesTreeOfBlobKey(String container, String blobKey) {
String normalizedBlobKey = denormalize(blobKey);
// exists is no path is present in the blobkey
if (!normalizedBlobKey.contains(File.separator)) return;
File file = new File(normalizedBlobKey);
// TODO
// "/media/data/works/java/amazon/jclouds/master/filesystem/aa/bb/cc/dd/eef6f0c8-0206-460b-8870-352e6019893c.txt"
String parentPath = file.getParent();
// no need to manage "/" parentPath, because "/" cannot be used as start
// char of blobkey
if (!isNullOrEmpty(parentPath)) {
// remove parent directory only it's empty
File directory = new File(buildPathStartingFromBaseDir(container, parentPath));
// don't delete directory if it's a directory blob
try {
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(directory.toPath());
if (view == null) { // OSX HFS+ does not support UserDefinedFileAttributeView
logger.debug("Could not look for attributes from %s", directory);
} else if (!view.list().isEmpty()) {
return;
}
} catch (IOException e) {
logger.debug("Could not look for attributes from %s: %s", directory, e);
}
String[] children = directory.list();
if (null == children || children.length == 0) {
try {
delete(directory);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", directory, e);
return;
}
// recursively call for removing other path
removeDirectoriesTreeOfBlobKey(container, parentPath);
}
}
}
private File openFolder(String folderName) throws IOException {
String baseFolderName = buildPathStartingFromBaseDir(folderName);
File folder = new File(baseFolderName);
if (folder.exists()) {
if (!folder.isDirectory()) {
throw new IOException("Resource " + baseFolderName + " isn't a folder.");
}
}
return folder;
}
private static void populateBlobKeysInContainer(
File directory, Set<String> blobNames, Function<String, String> function) {
File[] children = directory.listFiles();
if (children == null) {
return;
}
for (File child : children) {
if (child.isFile()) {
blobNames.add(function.apply(child.getAbsolutePath()));
} else if (child.isDirectory()) {
blobNames.add(
function.apply(child.getAbsolutePath()) + File.separator); // TODO: undo if failures
populateBlobKeysInContainer(child, blobNames, function);
}
}
}
/**
* Creates a directory and returns the result
*
* @param container
* @param directory
* @return true if the directory was created, otherwise false
*/
protected boolean createDirectoryWithResult(String container, String directory) {
String directoryFullName = buildPathStartingFromBaseDir(container, directory);
logger.debug("Creating directory %s", directoryFullName);
// cannot use directoryFullName, because the following method rebuild
// another time the path starting from base directory
if (buildPathAndChecksIfDirectoryExists(container, directory)) {
logger.debug("Directory %s already exists", directoryFullName);
return false;
}
File directoryToCreate = new File(directoryFullName);
boolean result = directoryToCreate.mkdirs();
return result;
}
/** Read the String representation of filesystem attribute, or return null if not present. */
private static String readStringAttributeIfPresent(
UserDefinedFileAttributeView view, Set<String> attributes, String name) throws IOException {
if (!attributes.contains(name)) {
return null;
}
ByteBuffer buf = ByteBuffer.allocate(view.size(name));
view.read(name, buf);
return new String(buf.array(), StandardCharsets.UTF_8);
}
/** Write an filesystem attribute, if its value is non-null. */
private static void writeStringAttributeIfPresent(
UserDefinedFileAttributeView view, String name, String value) throws IOException {
if (value != null) {
view.write(name, ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)));
}
}
private static void copyStringAttributeIfPresent(
UserDefinedFileAttributeView view, String name, Map<String, String> attrs)
throws IOException {
writeStringAttributeIfPresent(view, name, attrs.get(name));
}
}