public static void main(String[] args) throws Exception { System.out.println("Starting Cache Client..."); int bucket; List<CacheServiceInterface> serverlist = new ArrayList<CacheServiceInterface>(); serverlist.add(new DistributedCacheService("http://localhost:3000")); serverlist.add(new DistributedCacheService("http://localhost:3001")); serverlist.add(new DistributedCacheService("http://localhost:3002")); CharSequence val; for (int key = 1; key <= 10; key++) { val = generateRandomChar(); bucket = Hashing.consistentHash( Hashing.md5().hashString(Integer.toString(key)), serverlist.size()); serverlist.get(bucket).put(key, val.toString()); System.out.println("put (key " + key + " and value " + val + ")" + " in bucket " + bucket); } for (int key = 1; key <= 10; key++) { bucket = Hashing.consistentHash( Hashing.md5().hashString(Integer.toString(key)), serverlist.size()); System.out.println( "get (key " + key + " and value " + serverlist.get(bucket).get(key) + ")" + "from bucket " + bucket); } System.out.println("Exiting Cache Client..."); }
// goodFastHash(128) uses Murmur3_128. Use the same epsilon bounds. public void testGoodFastHash128() { HashTestUtils.check2BitAvalanche(Hashing.goodFastHash(128), 250, 0.20); HashTestUtils.checkAvalanche(Hashing.goodFastHash(128), 250, 0.17); HashTestUtils.checkNo2BitCharacteristics(Hashing.goodFastHash(128)); HashTestUtils.checkNoFunnels(Hashing.goodFastHash(128)); HashTestUtils.assertInvariants(Hashing.goodFastHash(128)); }
@Test public void shouldSetUpAPrivateLibraryIfGivenAMavenCoordWithoutDeps() throws Exception { resolver.resolve("com.example:no-deps:jar:1.0"); Path groupDir = thirdParty.resolve("example"); assertTrue(Files.exists(groupDir)); Path original = repo.resolve("com/example/no-deps/1.0/no-deps-1.0.jar"); HashCode expected = MorePaths.asByteSource(original).hash(Hashing.sha1()); Path jarFile = groupDir.resolve("no-deps-1.0.jar"); HashCode seen = MorePaths.asByteSource(jarFile).hash(Hashing.sha1()); assertEquals(expected, seen); List<Map<String, Object>> rules = buildFileParser.getAll(groupDir.resolve("BUCK")); assertEquals(1, rules.size()); Map<String, Object> rule = rules.get(0); // Name is derived from the project identifier assertEquals("no-deps", rule.get("name")); // The binary jar should be set assertEquals("no-deps-1.0.jar", rule.get("binaryJar")); // There was no source jar in the repo assertTrue(rule.containsKey("sourceJar")); assertNull(rule.get("sourceJar")); // Nothing depends on this, so it's not visible assertEquals(ImmutableList.of(), rule.get("visibility")); // And it doesn't depend on anything assertEquals(ImmutableList.of(), rule.get("deps")); }
public void testMurmur3_32() { HashTestUtils.check2BitAvalanche(Hashing.murmur3_32(), 250, 0.20); HashTestUtils.checkAvalanche(Hashing.murmur3_32(), 250, 0.17); HashTestUtils.checkNo2BitCharacteristics(Hashing.murmur3_32()); HashTestUtils.checkNoFunnels(Hashing.murmur3_32()); HashTestUtils.assertInvariants(Hashing.murmur3_32()); }
@Test public void testGeneratesDifferentIdenticonForDifferentHash() { byte[] hash = Hashing.sha256().hashString("test", UTF_8).asBytes(); byte[] hash2 = Hashing.sha256().hashString("test2", UTF_8).asBytes(); BufferedImage image = identicon.generate(hash); BufferedImage image2 = identicon.generate(hash2); assertThat(getImageBytes(image)).isNotEqualTo(getImageBytes(image2)); }
public void testCombineUnordered() { HashCode hash31 = HashCodes.fromInt(31); HashCode hash32 = HashCodes.fromInt(32); assertEquals(hash32, Hashing.combineUnordered(ImmutableList.of(hash32))); assertEquals(HashCodes.fromInt(64), Hashing.combineUnordered(ImmutableList.of(hash32, hash32))); assertEquals( HashCodes.fromInt(96), Hashing.combineUnordered(ImmutableList.of(hash32, hash32, hash32))); assertEquals( Hashing.combineUnordered(ImmutableList.of(hash31, hash32)), Hashing.combineUnordered(ImmutableList.of(hash32, hash31))); }
@Test public void mappedServersTest() { Map<Integer, String> servers = new HashMap<>(); servers.put(0, "server0"); servers.put(1, "server1"); HashFunction md5 = Hashing.md5(); List<PartitionEntry> triggers = generateTriggers(3, 1000); Map<Integer, String> newPartition; Map<Integer, String> oldPartition; print("initial - test 2 servers " + servers.toString()); newPartition = new HashMap<>(); for (PartitionEntry trigger : triggers) { newPartition.put( trigger.hashCode(), servers.get(Hashing.consistentHash(md5.hashInt(trigger.hashCode()), 2))); } for (int buckets = 3; buckets < 10; buckets++) { servers.put(buckets - 1, "server" + (buckets - 1)); print("test " + buckets + " servers " + servers.toString()); oldPartition = newPartition; newPartition = new HashMap<>(); for (PartitionEntry trigger : triggers) { newPartition.put( trigger.hashCode(), servers.get(Hashing.consistentHash(md5.hashInt(trigger.hashCode()), buckets))); } int changes = comparePartitions(oldPartition, newPartition); print( "Changes from " + (buckets - 1) + " to " + buckets + " servers: " + changes + " of " + oldPartition.size()); print("" + (((float) changes / (float) oldPartition.size()) * 100) + " % moved"); print( "K(" + oldPartition.size() + ")/n(" + buckets + "): " + ((float) oldPartition.size() / (float) buckets)); } }
public void testCombineUnordered_randomHashCodes() { Random random = new Random(); List<HashCode> hashCodes = Lists.newArrayList(); for (int i = 0; i < 10; i++) { hashCodes.add(HashCodes.fromLong(random.nextLong())); } HashCode hashCode1 = Hashing.combineUnordered(hashCodes); Collections.shuffle(hashCodes); HashCode hashCode2 = Hashing.combineUnordered(hashCodes); assertEquals(hashCode1, hashCode2); }
private String generateHash(@Nonnull List<TableRow> rows) { List<HashCode> rowHashes = Lists.newArrayList(); for (TableRow row : rows) { List<String> cellsInOneRow = Lists.newArrayList(); for (TableCell cell : row.getF()) { cellsInOneRow.add(Objects.toString(cell.getV())); Collections.sort(cellsInOneRow); } rowHashes.add(Hashing.sha1().hashString(cellsInOneRow.toString(), StandardCharsets.UTF_8)); } return Hashing.combineUnordered(rowHashes).toString(); }
public void testCombineOrdered() { HashCode hash31 = HashCodes.fromInt(31); HashCode hash32 = HashCodes.fromInt(32); assertEquals(hash32, Hashing.combineOrdered(ImmutableList.of(hash32))); assertEquals( HashCodes.fromBytes(new byte[] {(byte) 0x80, 0, 0, 0}), Hashing.combineOrdered(ImmutableList.of(hash32, hash32))); assertEquals( HashCodes.fromBytes(new byte[] {(byte) 0xa0, 0, 0, 0}), Hashing.combineOrdered(ImmutableList.of(hash32, hash32, hash32))); assertFalse( Hashing.combineOrdered(ImmutableList.of(hash31, hash32)) .equals(Hashing.combineOrdered(ImmutableList.of(hash32, hash31)))); }
public static void main(String[] args) throws Exception { System.out.println("Starting Cache Client..."); String[] value = { "0", "Ferrari 250 GTO", "Ferrari 250 Testa Rossa", "Jaguar XJ13", "Mercedes-Benz SLR McLaren", "Ferrari 330 P4", "Maybach Exelero", "Rolls-Royce Hyperion", "Lamborghini Veneno", "Zenvo ST1", "Audi Le Mans Concept", " McLaren X-1 Concept", "Koenigsegg CCXR Trevita" }; List<DistributedCacheService> server = new ArrayList<DistributedCacheService>(); server.add(new DistributedCacheService("http://localhost:3000")); server.add(new DistributedCacheService("http://localhost:3001")); server.add(new DistributedCacheService("http://localhost:3002")); System.out.println(" ----------------------Putting values to server------------------"); for (int putkey = 1; putkey <= 12; putkey++) { int bucket = Hashing.consistentHash(Hashing.md5().hashString(Integer.toString(putkey)), server.size()); server.get(bucket).put(putkey, value[putkey]); System.out.println( "The key value pair " + putkey + "-" + value[putkey] + " is assigned to server " + bucket); } System.out.println(" ----------------------Getting values from server------------------"); for (int getkey = 1; getkey <= 12; getkey++) { int bucket = Hashing.consistentHash(Hashing.md5().hashString(Integer.toString(getkey)), server.size()); System.out.println( "The key value pair " + getkey + "-" + server.get(bucket).get(getkey) + " is received to server " + bucket); } System.out.println(" ------------------------------Terminated---------------------------"); }
public HashCode getOutputHash(FileHashCache fileHashCache) throws IOException { Hasher hasher = Hashing.md5().newHasher(); for (Path path : getRecordedPaths()) { hasher.putBytes(fileHashCache.get(path).asBytes()); } return hasher.hash(); }
@Test public void testOnConnected() throws Exception { String password = "******"; when(userService.getPassword()).thenReturn(password); instance.connect(); botStartedFuture.get(TIMEOUT, TIMEOUT_UNIT); CountDownLatch latch = new CountDownLatch(1); doAnswer( invocation -> { latch.countDown(); return null; }) .when(outputIrc) .joinChannel(DEFAULT_CHANNEL_NAME); mockTaskService(); instance.connectionStateProperty().set(ConnectionState.CONNECTED); String md5Password = Hashing.md5().hashString(password, StandardCharsets.UTF_8).toString(); verify(outputIrc).message("NICKSERV", String.format("IDENTIFY %s", md5Password)); assertTrue("Channel has not been joined within timeout", latch.await(TIMEOUT, TIMEOUT_UNIT)); }
public static Inet4Address getCoercedIPv4Address(InetAddress paramInetAddress) { if ((paramInetAddress instanceof Inet4Address)) { return (Inet4Address) paramInetAddress; } byte[] arrayOfByte = paramInetAddress.getAddress(); int i = 1; for (int j = 0; j < 15; j++) { if (arrayOfByte[j] != 0) { i = 0; break; } } if ((i != 0) && (arrayOfByte[15] == 1)) { return LOOPBACK4; } if ((i != 0) && (arrayOfByte[15] == 0)) { return ANY4; } Inet6Address localInet6Address = (Inet6Address) paramInetAddress; long l = 0L; if (hasEmbeddedIPv4ClientAddress(localInet6Address)) { l = getEmbeddedIPv4ClientAddress(localInet6Address).hashCode(); } else { l = ByteBuffer.wrap(localInet6Address.getAddress(), 0, 8).getLong(); } int k = Hashing.murmur3_32().hashLong(l).asInt(); k |= 0xE0000000; if (k == -1) { k = -2; } return getInet4Address(Ints.toByteArray(k)); }
public static void main(String[] args) throws IOException { Closer closer = Closer.create(); // copy a file File origin = new File("join_temp"); File copy = new File("target_temp"); try { BufferedReader reader = new BufferedReader(new FileReader("join_temp")); BufferedWriter writer = new BufferedWriter(new FileWriter("target_temp")); closer.register(reader); closer.register(writer); String line; while ((line = reader.readLine()) != null) { writer.write(line); } } catch (IOException e) { throw closer.rethrow(e); } finally { closer.close(); } Files.copy(origin, copy); File moved = new File("moved"); // moving renaming Files.move(copy, moved); // working files as string List<String> lines = Files.readLines(origin, Charsets.UTF_8); HashCode hashCode = Files.hash(origin, Hashing.md5()); System.out.println(hashCode); // file write and append String hamlet = "To be, or not to be it is a question\n"; File write_and_append = new File("write_and_append"); Files.write(hamlet, write_and_append, Charsets.UTF_8); Files.append(hamlet, write_and_append, Charsets.UTF_8); // write_and_append.deleteOnExit(); Files.write("OverWrite the file", write_and_append, Charsets.UTF_8); // ByteSource ByteSink ByteSource fileBytes = Files.asByteSource(write_and_append); byte[] readBytes = fileBytes.read(); // equals to pre line -> Files.toByteArray(write_and_append) == readBytes ByteSink fileByteSink = Files.asByteSink(write_and_append); fileByteSink.write(Files.toByteArray(write_and_append)); BaseEncoding base64 = BaseEncoding.base64(); System.out.println(base64.encode("123456".getBytes())); }
public static Result processFile(Path prev, Path cur, Path outDir, int num, int gzipFrom) throws IOException { Result deltaHashes = new Result(); Path deltaFile = outDir.resolve(cur.getFileName().toString() + ".bpatch"); deleteIfExists(deltaFile); deltaHashes.path = deltaFile; boolean isGzipping = num >= gzipFrom; try (HashingOutputStream hashingStream = new HashingOutputStream( Hashing.sha256(), new BufferedOutputStream(newOutputStream(deltaFile, StandardOpenOption.CREATE_NEW)))) { GZIPOutputStream zipStream = null; GDiffWriter writer; if (isGzipping) { // Just constructing this object writes to the stream. zipStream = new GZIPOutputStream(hashingStream); writer = new GDiffWriter(zipStream); } else { writer = new GDiffWriter(hashingStream); } Delta delta = new Delta(); deltaHashes.preHash = sha256(readAllBytes(prev)); delta.compute(prev.toFile(), cur.toFile(), writer); if (isGzipping) zipStream.close(); deltaHashes.patchHash = hashingStream.hash().asBytes(); deltaHashes.postHash = sha256(readAllBytes(cur)); } long size = Files.size(deltaFile); deltaHashes.patchSize = size; println("... done: %s (%.2fkb) %s", deltaFile, size / 1024.0, isGzipping ? "zipped" : ""); return deltaHashes; }
/** Splits input by hashing the key. */ public class HashingSharder implements Sharder { private static final long serialVersionUID = 7967187256546710108L; private static final HashFunction HASH = Hashing.murmur3_32(); private int numShards; public HashingSharder(int numShards) { this.numShards = numShards; checkArgument(numShards > 0); } @Override public int getNumShards() { return numShards; } @Override public int getShardForKey(ByteBuffer key) { byte[] bytes = SerializationUtil.getBytes(key); int hash = (HASH.hashBytes(bytes).asInt()) & Integer.MAX_VALUE; // Keeping positive // Dividing integer range rather than using modulo so as to avoid rewriting entries if they are // re-hashed. return hash / (Integer.MAX_VALUE / numShards + 1); } }
private void addDirectoryToZipEntryList( File directory, String currentPath, ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder) throws IOException { Preconditions.checkNotNull(currentPath); for (File inputFile : directory.listFiles()) { String childPath = currentPath + (currentPath.isEmpty() ? "" : "/") + inputFile.getName(); if (inputFile.isDirectory()) { addDirectoryToZipEntryList(inputFile, childPath, zipEntriesBuilder); } else { ZipEntry nextEntry = new ZipEntry(childPath); long fileLength = inputFile.length(); if (fileLength > maxDeflatedBytes || EXTENSIONS_NOT_TO_DEFLATE.contains(Files.getFileExtension(inputFile.getName()))) { nextEntry.setMethod(ZipEntry.STORED); nextEntry.setCompressedSize(inputFile.length()); nextEntry.setSize(inputFile.length()); HashCode crc = ByteStreams.hash(Files.newInputStreamSupplier(inputFile), Hashing.crc32()); nextEntry.setCrc(crc.padToLong()); } zipEntriesBuilder.put(inputFile, nextEntry); } } }
public String generateAttributesHash(HttpRequestContext request) { StringBuilder attr = new StringBuilder(); attr.append(request.getHeaderValue("User-Agent")); // attr.append(request.getRequestUri().getHost()); TODO: add remote ip address after you // find how to get remote IP from HttpContext return Hashing.sha256().hashString(attr.toString(), StandardCharsets.UTF_8).toString(); }
public void testCombineUnordered_null() { try { Hashing.combineUnordered(null); fail(); } catch (NullPointerException expected) { } }
/** * Constructor. Initializes the RLBS Bloom Filter. * * @param k number of simple Bloom Filters composing this instance */ public RLBSBloomFilter(int k, int m, double thresFPR) { // Initialize parameters and calculate derived ones this.thresFPR = thresFPR; this.m = m; this.k = k; this.bitSetSize = (int) (m / k); // Prepare the hash functions to map items to positions in the bit array this.arrHashFunctions = new HashFunction[k]; this.arrBitSets = new BitSet[k]; for (int i = 0; i < k; i++) { // Murmur3 hashing functions, having different seeds are independent from each other this.arrHashFunctions[i] = Hashing.murmur3_128(i); // Each bit array implements a memory of m/k bit positions this.arrBitSets[i] = new BitSet(this.bitSetSize); } // Compute suggessted k, according to Bera et al. (pg. 24) double computedK = (Math.log(this.thresFPR) / Math.log(1 - (1 / Math.E))); logger.info( "RLBSBF initialized. Memory size (m): {}, Hash-functions: {}, Suggested Hash-functions: {}", this.m, this.k, computedK); }
@Override public void scrubFile(FileChannel file) throws IOException, ScrubException { if (!Machos.isMacho(file)) { return; } long size = file.size(); MappedByteBuffer map = file.map(FileChannel.MapMode.READ_WRITE, 0, size); try { Machos.setUuid(map, ZERO_UUID); } catch (Machos.MachoException e) { throw new ScrubException(e.getMessage()); } map.rewind(); Hasher hasher = Hashing.sha1().newHasher(); while (map.hasRemaining()) { hasher.putByte(map.get()); } map.rewind(); try { Machos.setUuid(map, Arrays.copyOf(hasher.hash().asBytes(), 16)); } catch (Machos.MachoException e) { throw new ScrubException(e.getMessage()); } }
public void testCombineUnordered_differentBitLengths() { try { Hashing.combineUnordered(ImmutableList.of(HashCodes.fromInt(32), HashCodes.fromLong(32L))); fail(); } catch (IllegalArgumentException expected) { } }
public void testGoodFastHash() { for (int i = 1; i < 200; i += 17) { HashFunction hasher = Hashing.goodFastHash(i); assertTrue(hasher.bits() >= i); HashTestUtils.assertInvariants(hasher); } }
/** Walk project references recursively, adding thrift files to the provided list. */ List<File> getRecursiveThriftFiles(MavenProject project, String outputDirectory, List<File> files) throws IOException { HashFunction hashFun = Hashing.md5(); if (dependencyIncludes.contains(project.getArtifactId())) { File dir = new File(new File(project.getFile().getParent(), "target"), outputDirectory); if (dir.exists()) { URI baseDir = getFileURI(dir); for (File f : findThriftFilesInDirectory(dir)) { URI fileURI = getFileURI(f); String relPath = baseDir.relativize(fileURI).getPath(); File destFolder = getResourcesOutputDirectory(); destFolder.mkdirs(); File destFile = new File(destFolder, relPath); if (!destFile.exists() || (destFile.isFile() && !Files.hash(f, hashFun).equals(Files.hash(destFile, hashFun)))) { getLog() .info( format("copying %s to %s", f.getCanonicalPath(), destFile.getCanonicalPath())); copyFile(f, destFile); } files.add(destFile); } } } Map<String, MavenProject> refs = project.getProjectReferences(); for (String name : refs.keySet()) { getRecursiveThriftFiles(refs.get(name), outputDirectory, files); } return files; }
public void sendUpUserMessage() throws IOException { Properties properties = new Properties(); properties.put("metadata.broker.list", "10.1.3.55:9092,10.1.3.56:9092,10.1.3.59:9092"); properties.put("serializer.class", "kafka.serializer.StringEncoder"); ProducerConfig producerConfig = new ProducerConfig(properties); kafka.javaapi.producer.Producer<String, String> producer = new kafka.javaapi.producer.Producer<String, String>(producerConfig); InputStream in = this.getClass().getResourceAsStream("/upusers.csv"); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); String line = null; while ((line = reader.readLine()) != null) { // System.out.println(line); ArrayList<String> list = new ArrayList<String>(Arrays.asList(line.replace(";NULL", "").split(","))); if (list.size() != 0) { list.remove(0); String uid = list.remove(0); String nline = Hashing.md5() .hashString( uid + System.currentTimeMillis() + new Random().nextLong(), Charsets.UTF_8) + "," + Joiner.on(",").join(list.toArray()).toString(); // String nline = Joiner.on(",").join(list.toArray()).toString(); KeyedMessage<String, String> message = new KeyedMessage<String, String>(TOPIC, nline); producer.send(message); // System.out.println(nline); // System.out.println(nline); } } }
@Test public void testComputeAbiKey() { ImmutableSortedMap<String, HashCode> classNamesAndHashes = ImmutableSortedMap.of( "com/example/Foo", HashCode.fromString("e4fccb7520b7795e632651323c63217c9f59f72a"), "com/example/Bar", HashCode.fromString("087b7707a5f8e0a2adf5652e3cd2072d89a197dc"), "com/example/Baz", HashCode.fromString("62b1c2510840c0de55c13f66065a98a719be0f19")); String observedSha1 = DexProducedFromJavaLibrary.computeAbiKey(classNamesAndHashes).getHash(); String expectedSha1 = Hashing.sha1() .newHasher() .putUnencodedChars("com/example/Bar") .putByte((byte) 0) .putUnencodedChars("087b7707a5f8e0a2adf5652e3cd2072d89a197dc") .putByte((byte) 0) .putUnencodedChars("com/example/Baz") .putByte((byte) 0) .putUnencodedChars("62b1c2510840c0de55c13f66065a98a719be0f19") .putByte((byte) 0) .putUnencodedChars("com/example/Foo") .putByte((byte) 0) .putUnencodedChars("e4fccb7520b7795e632651323c63217c9f59f72a") .putByte((byte) 0) .hash() .toString(); assertEquals(expectedSha1, observedSha1); }
public void testCombineUnordered_empty() { try { Hashing.combineUnordered(Collections.<HashCode>emptySet()); fail(); } catch (IllegalArgumentException expected) { } }
/** * Non-idempotent RuleKeys are normally output as strings of 'x' characters, but when comparing * two sets of RuleKeys in textual form it is necessary to mangle one of the two sets, so that * non-idempotent RuleKeys are never considered equal. */ public String toString(boolean mangleNonIdempotent) { if (!isIdempotent()) { return new String(new char[Hashing.sha1().bits() / 4]) .replace("\0", mangleNonIdempotent ? "y" : "x"); } return hashCode.toString(); }
private static final class PeriodSelector<K> implements Function<K, Period> { private static final HashFunction hasher = Hashing.goodFastHash(Integer.SIZE); private final ImmutableList<Period> periods; private final long totalSize; public PeriodSelector(final Iterable<? extends Period> periods) { this.periods = ImmutableList.copyOf(periods); long size = 0L; for (final Period period : this.periods) { size += period.size; } this.totalSize = size; } /** * Imagine all of the periods provided in the constructor laid end to end on a number line, each * the length of its size, stretching from zero to the sum of all of the sizes. Hash the given * key and mod it with that sum to yield a position along that number line. Then identify the * period that covers that portion of the value. This approach has the benefit of being both * fast and deterministic for a given key. */ @Override public Period apply(final K elementKey) { // Rehash for better bit dispersion: final int hashCode = hasher.hashInt(elementKey.hashCode()).asInt(); final long value = hashCode % this.totalSize; int i = 0; for (int bound = 0; i < this.periods.size() && value >= (bound += this.periods.get(i).size); ++i) { // do nothing } return this.periods.get(i); } }