@Override
  public void scrubFile(FileChannel file) throws IOException, ScrubException {
    if (!Machos.isMacho(file)) {
      return;
    }

    long size = file.size();
    MappedByteBuffer map = file.map(FileChannel.MapMode.READ_WRITE, 0, size);

    try {
      Machos.setUuid(map, ZERO_UUID);
    } catch (Machos.MachoException e) {
      throw new ScrubException(e.getMessage());
    }
    map.rewind();

    Hasher hasher = Hashing.sha1().newHasher();
    while (map.hasRemaining()) {
      hasher.putByte(map.get());
    }

    map.rewind();
    try {
      Machos.setUuid(map, Arrays.copyOf(hasher.hash().asBytes(), 16));
    } catch (Machos.MachoException e) {
      throw new ScrubException(e.getMessage());
    }
  }
Пример #2
0
 @Override
 public void hashRow(DimensionSelector dimSelector, Hasher hasher) {
   final IndexedInts row = dimSelector.getRow();
   final int size = row.size();
   // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases.
   if (size == 1) {
     final String value = dimSelector.lookupName(row.get(0));
     hasher.putUnencodedChars(nullToSpecial(value));
   } else if (size != 0) {
     final String[] values = new String[size];
     for (int i = 0; i < size; ++i) {
       final String value = dimSelector.lookupName(row.get(i));
       values[i] = nullToSpecial(value);
     }
     // Values need to be sorted to ensure consistent multi-value ordering across different
     // segments
     Arrays.sort(values);
     for (int i = 0; i < size; ++i) {
       if (i != 0) {
         hasher.putChar(CARDINALITY_AGG_SEPARATOR);
       }
       hasher.putUnencodedChars(values[i]);
     }
   }
 }
Пример #3
0
 public HashCode getOutputHash(FileHashCache fileHashCache) throws IOException {
   Hasher hasher = Hashing.md5().newHasher();
   for (Path path : getRecordedPaths()) {
     hasher.putBytes(fileHashCache.get(path).asBytes());
   }
   return hasher.hash();
 }
Пример #4
0
 @Override
 public int hashCode() {
   HashFunction hf = Hashing.murmur3_32();
   Hasher hc = hf.newHasher();
   for (String key : fields.keySet()) {
     hc.putString(key, Charsets.UTF_8);
   }
   return hc.hash().asInt();
 }
Пример #5
0
 private static HashCode hashPath(Path file, final Hasher hasher) throws IOException {
   byte[] tmpBuffer = new byte[512];
   final InputStream in = Files.newInputStream(file);
   for (int read = in.read(tmpBuffer); read > 0; read = in.read(tmpBuffer)) {
     hasher.putBytes(tmpBuffer, 0, read);
   }
   final HashCode fileHash = hasher.hash();
   in.close();
   return fileHash;
 }
 @Override
 public int read(@Nonnull final byte[] bytes, final int off, final int len) throws IOException {
   int numRead = in.read(bytes, off, len);
   if (numRead != -1) {
     for (Hasher hasher : hashers.values()) {
       hasher.putBytes(bytes, off, numRead);
     }
     count += numRead;
   }
   return numRead;
 }
 @Override
 public int read() throws IOException {
   int b = in.read();
   if (b != -1) {
     for (Hasher hasher : hashers.values()) {
       hasher.putByte((byte) b);
     }
     count++;
   }
   return b;
 }
Пример #8
0
  private void deserializeRule(DeserializationContext context, Build.Rule rulePb)
      throws PackageDeserializationException, InterruptedException {
    Location ruleLocation = EmptyLocation.INSTANCE;
    RuleClass ruleClass = packageDeserializationEnvironment.getRuleClass(rulePb, ruleLocation);
    Map<String, ParsedAttributeValue> attributeValues = new HashMap<>();
    AttributesToDeserialize attrToDeserialize =
        packageDeserializationEnvironment.attributesToDeserialize();

    Hasher hasher = Hashing.md5().newHasher();
    for (Build.Attribute attrPb : rulePb.getAttributeList()) {
      Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType();
      attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb));
      if (attrToDeserialize.addSyntheticAttributeHash) {
        // TODO(bazel-team): This might give false positives because of explicit vs implicit.
        hasher.putBytes(attrPb.toByteArray());
      }
    }
    AttributeContainerWithoutLocation attributeContainer =
        new AttributeContainerWithoutLocation(ruleClass, hasher.hash());

    Label ruleLabel = deserializeLabel(rulePb.getName());
    try {
      Rule rule =
          createRuleWithParsedAttributeValues(
              ruleClass,
              ruleLabel,
              context.packageBuilder,
              ruleLocation,
              attributeValues,
              NullEventHandler.INSTANCE,
              attributeContainer);
      context.packageBuilder.addRule(rule);

      // Remove the attribute after it is added to package in order to pass the validations
      // and be able to compute all the outputs.
      if (attrToDeserialize != DESERIALIZE_ALL_ATTRS) {
        for (String attrName : attributeValues.keySet()) {
          Attribute attribute = ruleClass.getAttributeByName(attrName);
          if (!(attrToDeserialize.shouldKeepAttributeWithName.apply(attrName)
              || BuildType.isLabelType(attribute.getType()))) {
            attributeContainer.clearIfNotLabel(attrName);
          }
        }
      }

      Preconditions.checkState(!rule.containsErrors());
    } catch (NameConflictException | LabelSyntaxException e) {
      throw new PackageDeserializationException(e);
    }
  }
 @Override
 public int hashCode() {
   HashFunction hf = Hashing.goodFastHash(32);
   Hasher h = hf.newHasher();
   h.putInt(slots.size());
   for (int i = 0; i < slots.size(); i++) {
     h.putInt(slots.get(i).size());
     for (int j = 0; j < slots.size(); j++) {
       h.putBytes(slots.get(i).get(j).getLowerRange());
       h.putBytes(slots.get(i).get(j).getUpperRange());
     }
   }
   return h.hash().asInt();
 }
Пример #10
0
 private static int fingerprint(
     final NetworkInfoSource source,
     final List<com.eucalyptus.cluster.Cluster> clusters,
     final Set<String> dirtyPublicAddresses,
     final String networkConfiguration) {
   final HashFunction hashFunction = goodFastHash(32);
   final Hasher hasher = hashFunction.newHasher();
   final Funnel<VersionedNetworkView> versionedItemFunnel =
       new Funnel<VersionedNetworkView>() {
         @Override
         public void funnel(final VersionedNetworkView o, final PrimitiveSink primitiveSink) {
           primitiveSink.putString(o.getId(), StandardCharsets.UTF_8);
           primitiveSink.putChar('=');
           primitiveSink.putInt(o.getVersion());
         }
       };
   for (final Map.Entry<String, Iterable<? extends VersionedNetworkView>> entry :
       source.getView().entrySet()) {
     hasher.putString(entry.getKey(), StandardCharsets.UTF_8);
     for (final VersionedNetworkView item : entry.getValue()) {
       hasher.putObject(item, versionedItemFunnel);
     }
   }
   hasher.putString(
       Joiner.on(',').join(Sets.newTreeSet(Iterables.transform(clusters, HasName.GET_NAME))),
       StandardCharsets.UTF_8);
   hasher.putString(
       Joiner.on(',').join(Sets.newTreeSet(dirtyPublicAddresses)), StandardCharsets.UTF_8);
   hasher.putInt(networkConfiguration.hashCode());
   return hasher.hash().asInt();
 }
Пример #11
0
 public RuleKey build() {
   RuleKey ruleKey = idempotent ? new RuleKey(hasher.hash()) : new RuleKey(null);
   if (logElms != null) {
     logger.info(
         String.format(
             "%sRuleKey %s=%s",
             ruleKey.isIdempotent() ? "" : "non-idempotent ",
             ruleKey.toString(),
             Joiner.on("").join(logElms)));
   }
   return ruleKey;
 }
Пример #12
0
  @Override
  protected HashCode hash(HashFunction hashFunction, Key k) {
    Hasher hasher = hashFunction.newHasher();

    if (row) {
      putByteSquence(k.getRowData(), hasher);
    }

    if (family) {
      putByteSquence(k.getColumnFamilyData(), hasher);
    }

    if (qualifier) {
      putByteSquence(k.getColumnQualifierData(), hasher);
    }

    if (visibility) {
      putByteSquence(k.getColumnVisibilityData(), hasher);
    }

    return hasher.hash();
  }
 /**
  * 创建注册中心.
  *
  * @param connectString 注册中心连接字符串
  * @param namespace 注册中心命名空间
  * @param digest 注册中心凭证
  * @return 注册中心对象
  */
 public static CoordinatorRegistryCenter createCoordinatorRegistryCenter(
     final String connectString, final String namespace, final Optional<String> digest) {
   Hasher hasher =
       Hashing.md5()
           .newHasher()
           .putString(connectString, Charsets.UTF_8)
           .putString(namespace, Charsets.UTF_8);
   if (digest.isPresent()) {
     hasher.putString(digest.get(), Charsets.UTF_8);
   }
   HashCode hashCode = hasher.hash();
   if (regCenterMap.containsKey(hashCode)) {
     return regCenterMap.get(hashCode);
   }
   ZookeeperConfiguration zkConfig = new ZookeeperConfiguration(connectString, namespace);
   if (digest.isPresent()) {
     zkConfig.setDigest(digest.get());
   }
   CoordinatorRegistryCenter result = new ZookeeperRegistryCenter(zkConfig);
   result.init();
   regCenterMap.putIfAbsent(hashCode, result);
   return result;
 }
 @VisibleForTesting
 static Sha1HashCode computeAbiKey(ImmutableSortedMap<String, HashCode> classNames) {
   Hasher hasher = Hashing.sha1().newHasher();
   for (Map.Entry<String, HashCode> entry : classNames.entrySet()) {
     hasher.putUnencodedChars(entry.getKey());
     hasher.putByte((byte) 0);
     hasher.putUnencodedChars(entry.getValue().toString());
     hasher.putByte((byte) 0);
   }
   return Sha1HashCode.fromHashCode(hasher.hash());
 }
Пример #15
0
  private String etag(MarkdownFile srcmd, @Nullable MarkdownFile navmd) {
    byte[] b = new byte[Constants.OBJECT_ID_LENGTH];
    Hasher h = Hashing.sha1().newHasher();
    h.putInt(ETAG_GEN);

    renderer.getTemplateHash(SOY_FILE).writeBytesTo(b, 0, b.length);
    h.putBytes(b);

    if (navmd != null) {
      navmd.id.copyRawTo(b, 0);
      h.putBytes(b);
    }

    srcmd.id.copyRawTo(b, 0);
    h.putBytes(b);
    return h.hash().toString();
  }
Пример #16
0
 private void updateSubmissionId(Change change) {
   Hasher h = Hashing.sha1().newHasher();
   h.putLong(Thread.currentThread().getId()).putUnencodedChars(MACHINE_ID);
   staticSubmissionId = h.hash().toString().substring(0, 8);
   submissionId = change.getId().get() + "-" + TimeUtil.nowMs() + "-" + staticSubmissionId;
 }
Пример #17
0
 private void putByteSquence(ByteSequence data, Hasher hasher) {
   hasher.putBytes(data.getBackingArray(), data.offset(), data.length());
 }
Пример #18
0
 /**
  * Hashes the contents of this byte source using the given hash function.
  *
  * @throws IOException if an I/O error occurs in the process of reading from this source
  */
 public HashCode hash(HashFunction hashFunction) throws IOException {
   Hasher hasher = hashFunction.newHasher();
   copyTo(Funnels.asOutputStream(hasher));
   return hasher.hash();
 }
Пример #19
0
 private <T extends Value> HashCode taskHash(Task task) {
   Hasher hasher = Hash.newHasher();
   hasher.putBytes(smoothJarHash.asBytes());
   hasher.putBytes(task.hash().asBytes());
   return hasher.hash();
 }
Пример #20
0
 private Builder separate() {
   hasher.putByte(SEPARATOR);
   return this;
 }
Пример #21
0
 private Builder feed(byte[] bytes) {
   hasher.putBytes(bytes);
   return this;
 }
  @SuppressWarnings({"rawtypes", "unchecked"})
  private TargetNode<?> createTargetNode(
      BuckEventBus eventBus,
      Cell cell,
      Path buildFile,
      BuildTarget target,
      Map<String, Object> rawNode,
      TargetNodeListener nodeListener) {
    BuildRuleType buildRuleType = parseBuildRuleTypeFromRawRule(cell, rawNode);

    // Because of the way that the parser works, we know this can never return null.
    Description<?> description = cell.getDescription(buildRuleType);

    if (target.isFlavored()) {
      if (description instanceof Flavored) {
        if (!((Flavored) description).hasFlavors(ImmutableSet.copyOf(target.getFlavors()))) {
          throw new HumanReadableException(
              "Unrecognized flavor in target %s while parsing %s%s.",
              target,
              UnflavoredBuildTarget.BUILD_TARGET_PREFIX,
              MorePaths.pathWithUnixSeparators(
                  target.getBasePath().resolve(cell.getBuildFileName())));
        }
      } else {
        LOG.warn(
            "Target %s (type %s) must implement the Flavored interface "
                + "before we can check if it supports flavors: %s",
            target.getUnflavoredBuildTarget(), buildRuleType, target.getFlavors());
        throw new HumanReadableException(
            "Target %s (type %s) does not currently support flavors (tried %s)",
            target.getUnflavoredBuildTarget(), buildRuleType, target.getFlavors());
      }
    }

    Cell targetCell = cell.getCell(target);
    BuildRuleFactoryParams factoryParams =
        new BuildRuleFactoryParams(
            targetCell.getFilesystem(),
            target.withoutCell(),
            new FilesystemBackedBuildFileTree(cell.getFilesystem(), cell.getBuildFileName()),
            targetCell.isEnforcingBuckPackageBoundaries());
    Object constructorArg = description.createUnpopulatedConstructorArg();
    try {
      ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
      ImmutableSet.Builder<BuildTargetPattern> visibilityPatterns = ImmutableSet.builder();
      try (SimplePerfEvent.Scope scope =
          SimplePerfEvent.scope(
              eventBus, PerfEventId.of("MarshalledConstructorArg"), "target", target)) {
        marshaller.populate(
            targetCell.getCellRoots(),
            targetCell.getFilesystem(),
            factoryParams,
            constructorArg,
            declaredDeps,
            visibilityPatterns,
            rawNode);
      }
      try (SimplePerfEvent.Scope scope =
          SimplePerfEvent.scope(eventBus, PerfEventId.of("CreatedTargetNode"), "target", target)) {
        Hasher hasher = Hashing.sha1().newHasher();
        hasher.putString(BuckVersion.getVersion(), UTF_8);
        JsonObjectHashing.hashJsonObject(hasher, rawNode);
        synchronized (this) {
          targetsCornucopia.put(target.getUnflavoredBuildTarget(), target);
        }
        TargetNode<?> node =
            new TargetNode(
                hasher.hash(),
                description,
                constructorArg,
                typeCoercerFactory,
                factoryParams,
                declaredDeps.build(),
                visibilityPatterns.build(),
                targetCell.getCellRoots());
        nodeListener.onCreate(buildFile, node);
        return node;
      }
    } catch (NoSuchBuildTargetException | TargetNode.InvalidSourcePathInputException e) {
      throw new HumanReadableException(e);
    } catch (ConstructorArgMarshalException e) {
      throw new HumanReadableException("%s: %s", target, e.getMessage());
    } catch (IOException e) {
      throw new HumanReadableException(e.getMessage(), e);
    }
  }