@Override
public void scrubFile(FileChannel file) throws IOException, ScrubException {
if (!Machos.isMacho(file)) {
return;
}
long size = file.size();
MappedByteBuffer map = file.map(FileChannel.MapMode.READ_WRITE, 0, size);
try {
Machos.setUuid(map, ZERO_UUID);
} catch (Machos.MachoException e) {
throw new ScrubException(e.getMessage());
}
map.rewind();
Hasher hasher = Hashing.sha1().newHasher();
while (map.hasRemaining()) {
hasher.putByte(map.get());
}
map.rewind();
try {
Machos.setUuid(map, Arrays.copyOf(hasher.hash().asBytes(), 16));
} catch (Machos.MachoException e) {
throw new ScrubException(e.getMessage());
}
}
@Override
public void hashRow(DimensionSelector dimSelector, Hasher hasher) {
final IndexedInts row = dimSelector.getRow();
final int size = row.size();
// nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases.
if (size == 1) {
final String value = dimSelector.lookupName(row.get(0));
hasher.putUnencodedChars(nullToSpecial(value));
} else if (size != 0) {
final String[] values = new String[size];
for (int i = 0; i < size; ++i) {
final String value = dimSelector.lookupName(row.get(i));
values[i] = nullToSpecial(value);
}
// Values need to be sorted to ensure consistent multi-value ordering across different
// segments
Arrays.sort(values);
for (int i = 0; i < size; ++i) {
if (i != 0) {
hasher.putChar(CARDINALITY_AGG_SEPARATOR);
}
hasher.putUnencodedChars(values[i]);
}
}
}
public HashCode getOutputHash(FileHashCache fileHashCache) throws IOException {
Hasher hasher = Hashing.md5().newHasher();
for (Path path : getRecordedPaths()) {
hasher.putBytes(fileHashCache.get(path).asBytes());
}
return hasher.hash();
}
@Override
public int hashCode() {
HashFunction hf = Hashing.murmur3_32();
Hasher hc = hf.newHasher();
for (String key : fields.keySet()) {
hc.putString(key, Charsets.UTF_8);
}
return hc.hash().asInt();
}
private static HashCode hashPath(Path file, final Hasher hasher) throws IOException {
byte[] tmpBuffer = new byte[512];
final InputStream in = Files.newInputStream(file);
for (int read = in.read(tmpBuffer); read > 0; read = in.read(tmpBuffer)) {
hasher.putBytes(tmpBuffer, 0, read);
}
final HashCode fileHash = hasher.hash();
in.close();
return fileHash;
}
@Override
public int read(@Nonnull final byte[] bytes, final int off, final int len) throws IOException {
int numRead = in.read(bytes, off, len);
if (numRead != -1) {
for (Hasher hasher : hashers.values()) {
hasher.putBytes(bytes, off, numRead);
}
count += numRead;
}
return numRead;
}
@Override
public int read() throws IOException {
int b = in.read();
if (b != -1) {
for (Hasher hasher : hashers.values()) {
hasher.putByte((byte) b);
}
count++;
}
return b;
}
private void deserializeRule(DeserializationContext context, Build.Rule rulePb)
throws PackageDeserializationException, InterruptedException {
Location ruleLocation = EmptyLocation.INSTANCE;
RuleClass ruleClass = packageDeserializationEnvironment.getRuleClass(rulePb, ruleLocation);
Map<String, ParsedAttributeValue> attributeValues = new HashMap<>();
AttributesToDeserialize attrToDeserialize =
packageDeserializationEnvironment.attributesToDeserialize();
Hasher hasher = Hashing.md5().newHasher();
for (Build.Attribute attrPb : rulePb.getAttributeList()) {
Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType();
attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb));
if (attrToDeserialize.addSyntheticAttributeHash) {
// TODO(bazel-team): This might give false positives because of explicit vs implicit.
hasher.putBytes(attrPb.toByteArray());
}
}
AttributeContainerWithoutLocation attributeContainer =
new AttributeContainerWithoutLocation(ruleClass, hasher.hash());
Label ruleLabel = deserializeLabel(rulePb.getName());
try {
Rule rule =
createRuleWithParsedAttributeValues(
ruleClass,
ruleLabel,
context.packageBuilder,
ruleLocation,
attributeValues,
NullEventHandler.INSTANCE,
attributeContainer);
context.packageBuilder.addRule(rule);
// Remove the attribute after it is added to package in order to pass the validations
// and be able to compute all the outputs.
if (attrToDeserialize != DESERIALIZE_ALL_ATTRS) {
for (String attrName : attributeValues.keySet()) {
Attribute attribute = ruleClass.getAttributeByName(attrName);
if (!(attrToDeserialize.shouldKeepAttributeWithName.apply(attrName)
|| BuildType.isLabelType(attribute.getType()))) {
attributeContainer.clearIfNotLabel(attrName);
}
}
}
Preconditions.checkState(!rule.containsErrors());
} catch (NameConflictException | LabelSyntaxException e) {
throw new PackageDeserializationException(e);
}
}
@Override
public int hashCode() {
HashFunction hf = Hashing.goodFastHash(32);
Hasher h = hf.newHasher();
h.putInt(slots.size());
for (int i = 0; i < slots.size(); i++) {
h.putInt(slots.get(i).size());
for (int j = 0; j < slots.size(); j++) {
h.putBytes(slots.get(i).get(j).getLowerRange());
h.putBytes(slots.get(i).get(j).getUpperRange());
}
}
return h.hash().asInt();
}
private static int fingerprint(
final NetworkInfoSource source,
final List<com.eucalyptus.cluster.Cluster> clusters,
final Set<String> dirtyPublicAddresses,
final String networkConfiguration) {
final HashFunction hashFunction = goodFastHash(32);
final Hasher hasher = hashFunction.newHasher();
final Funnel<VersionedNetworkView> versionedItemFunnel =
new Funnel<VersionedNetworkView>() {
@Override
public void funnel(final VersionedNetworkView o, final PrimitiveSink primitiveSink) {
primitiveSink.putString(o.getId(), StandardCharsets.UTF_8);
primitiveSink.putChar('=');
primitiveSink.putInt(o.getVersion());
}
};
for (final Map.Entry<String, Iterable<? extends VersionedNetworkView>> entry :
source.getView().entrySet()) {
hasher.putString(entry.getKey(), StandardCharsets.UTF_8);
for (final VersionedNetworkView item : entry.getValue()) {
hasher.putObject(item, versionedItemFunnel);
}
}
hasher.putString(
Joiner.on(',').join(Sets.newTreeSet(Iterables.transform(clusters, HasName.GET_NAME))),
StandardCharsets.UTF_8);
hasher.putString(
Joiner.on(',').join(Sets.newTreeSet(dirtyPublicAddresses)), StandardCharsets.UTF_8);
hasher.putInt(networkConfiguration.hashCode());
return hasher.hash().asInt();
}
public RuleKey build() {
RuleKey ruleKey = idempotent ? new RuleKey(hasher.hash()) : new RuleKey(null);
if (logElms != null) {
logger.info(
String.format(
"%sRuleKey %s=%s",
ruleKey.isIdempotent() ? "" : "non-idempotent ",
ruleKey.toString(),
Joiner.on("").join(logElms)));
}
return ruleKey;
}
@Override
protected HashCode hash(HashFunction hashFunction, Key k) {
Hasher hasher = hashFunction.newHasher();
if (row) {
putByteSquence(k.getRowData(), hasher);
}
if (family) {
putByteSquence(k.getColumnFamilyData(), hasher);
}
if (qualifier) {
putByteSquence(k.getColumnQualifierData(), hasher);
}
if (visibility) {
putByteSquence(k.getColumnVisibilityData(), hasher);
}
return hasher.hash();
}
/**
* 创建注册中心.
*
* @param connectString 注册中心连接字符串
* @param namespace 注册中心命名空间
* @param digest 注册中心凭证
* @return 注册中心对象
*/
public static CoordinatorRegistryCenter createCoordinatorRegistryCenter(
final String connectString, final String namespace, final Optional<String> digest) {
Hasher hasher =
Hashing.md5()
.newHasher()
.putString(connectString, Charsets.UTF_8)
.putString(namespace, Charsets.UTF_8);
if (digest.isPresent()) {
hasher.putString(digest.get(), Charsets.UTF_8);
}
HashCode hashCode = hasher.hash();
if (regCenterMap.containsKey(hashCode)) {
return regCenterMap.get(hashCode);
}
ZookeeperConfiguration zkConfig = new ZookeeperConfiguration(connectString, namespace);
if (digest.isPresent()) {
zkConfig.setDigest(digest.get());
}
CoordinatorRegistryCenter result = new ZookeeperRegistryCenter(zkConfig);
result.init();
regCenterMap.putIfAbsent(hashCode, result);
return result;
}
@VisibleForTesting
static Sha1HashCode computeAbiKey(ImmutableSortedMap<String, HashCode> classNames) {
Hasher hasher = Hashing.sha1().newHasher();
for (Map.Entry<String, HashCode> entry : classNames.entrySet()) {
hasher.putUnencodedChars(entry.getKey());
hasher.putByte((byte) 0);
hasher.putUnencodedChars(entry.getValue().toString());
hasher.putByte((byte) 0);
}
return Sha1HashCode.fromHashCode(hasher.hash());
}
private String etag(MarkdownFile srcmd, @Nullable MarkdownFile navmd) {
byte[] b = new byte[Constants.OBJECT_ID_LENGTH];
Hasher h = Hashing.sha1().newHasher();
h.putInt(ETAG_GEN);
renderer.getTemplateHash(SOY_FILE).writeBytesTo(b, 0, b.length);
h.putBytes(b);
if (navmd != null) {
navmd.id.copyRawTo(b, 0);
h.putBytes(b);
}
srcmd.id.copyRawTo(b, 0);
h.putBytes(b);
return h.hash().toString();
}
private void updateSubmissionId(Change change) {
Hasher h = Hashing.sha1().newHasher();
h.putLong(Thread.currentThread().getId()).putUnencodedChars(MACHINE_ID);
staticSubmissionId = h.hash().toString().substring(0, 8);
submissionId = change.getId().get() + "-" + TimeUtil.nowMs() + "-" + staticSubmissionId;
}
private void putByteSquence(ByteSequence data, Hasher hasher) {
hasher.putBytes(data.getBackingArray(), data.offset(), data.length());
}
/**
* Hashes the contents of this byte source using the given hash function.
*
* @throws IOException if an I/O error occurs in the process of reading from this source
*/
public HashCode hash(HashFunction hashFunction) throws IOException {
Hasher hasher = hashFunction.newHasher();
copyTo(Funnels.asOutputStream(hasher));
return hasher.hash();
}
private <T extends Value> HashCode taskHash(Task task) {
Hasher hasher = Hash.newHasher();
hasher.putBytes(smoothJarHash.asBytes());
hasher.putBytes(task.hash().asBytes());
return hasher.hash();
}
private Builder separate() {
hasher.putByte(SEPARATOR);
return this;
}
private Builder feed(byte[] bytes) {
hasher.putBytes(bytes);
return this;
}
@SuppressWarnings({"rawtypes", "unchecked"})
private TargetNode<?> createTargetNode(
BuckEventBus eventBus,
Cell cell,
Path buildFile,
BuildTarget target,
Map<String, Object> rawNode,
TargetNodeListener nodeListener) {
BuildRuleType buildRuleType = parseBuildRuleTypeFromRawRule(cell, rawNode);
// Because of the way that the parser works, we know this can never return null.
Description<?> description = cell.getDescription(buildRuleType);
if (target.isFlavored()) {
if (description instanceof Flavored) {
if (!((Flavored) description).hasFlavors(ImmutableSet.copyOf(target.getFlavors()))) {
throw new HumanReadableException(
"Unrecognized flavor in target %s while parsing %s%s.",
target,
UnflavoredBuildTarget.BUILD_TARGET_PREFIX,
MorePaths.pathWithUnixSeparators(
target.getBasePath().resolve(cell.getBuildFileName())));
}
} else {
LOG.warn(
"Target %s (type %s) must implement the Flavored interface "
+ "before we can check if it supports flavors: %s",
target.getUnflavoredBuildTarget(), buildRuleType, target.getFlavors());
throw new HumanReadableException(
"Target %s (type %s) does not currently support flavors (tried %s)",
target.getUnflavoredBuildTarget(), buildRuleType, target.getFlavors());
}
}
Cell targetCell = cell.getCell(target);
BuildRuleFactoryParams factoryParams =
new BuildRuleFactoryParams(
targetCell.getFilesystem(),
target.withoutCell(),
new FilesystemBackedBuildFileTree(cell.getFilesystem(), cell.getBuildFileName()),
targetCell.isEnforcingBuckPackageBoundaries());
Object constructorArg = description.createUnpopulatedConstructorArg();
try {
ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
ImmutableSet.Builder<BuildTargetPattern> visibilityPatterns = ImmutableSet.builder();
try (SimplePerfEvent.Scope scope =
SimplePerfEvent.scope(
eventBus, PerfEventId.of("MarshalledConstructorArg"), "target", target)) {
marshaller.populate(
targetCell.getCellRoots(),
targetCell.getFilesystem(),
factoryParams,
constructorArg,
declaredDeps,
visibilityPatterns,
rawNode);
}
try (SimplePerfEvent.Scope scope =
SimplePerfEvent.scope(eventBus, PerfEventId.of("CreatedTargetNode"), "target", target)) {
Hasher hasher = Hashing.sha1().newHasher();
hasher.putString(BuckVersion.getVersion(), UTF_8);
JsonObjectHashing.hashJsonObject(hasher, rawNode);
synchronized (this) {
targetsCornucopia.put(target.getUnflavoredBuildTarget(), target);
}
TargetNode<?> node =
new TargetNode(
hasher.hash(),
description,
constructorArg,
typeCoercerFactory,
factoryParams,
declaredDeps.build(),
visibilityPatterns.build(),
targetCell.getCellRoots());
nodeListener.onCreate(buildFile, node);
return node;
}
} catch (NoSuchBuildTargetException | TargetNode.InvalidSourcePathInputException e) {
throw new HumanReadableException(e);
} catch (ConstructorArgMarshalException e) {
throw new HumanReadableException("%s: %s", target, e.getMessage());
} catch (IOException e) {
throw new HumanReadableException(e.getMessage(), e);
}
}
