コード例 #1
0
  /* decrypt, then compute
   */
  @Override
  public void map(BytesWritable key, BytesWritable val, Context context)
      throws IOException, InterruptedException {
    byte[] iv = key.copyBytes();
    byte[] ct = val.copyBytes();
    // String[] ss = new String(crypto.decrypt_word_rnd(ct, iv)).split("\\s+");

    String text = new String(crypto.decrypt_word_rnd(ct, iv));
    Matcher matcher = pattern.matcher(text);

    while (matcher.find()) {
      context.write(new Text(matcher.group(0)), new LongWritable(1));
    }
  }
コード例 #2
0
  @Override
  protected void map(Writable key, BytesWritable value, Context context)
      throws IOException, InterruptedException {

    int percentOfWritten = context.getConfiguration().getInt("percentOfWritten", 100);

    DocumentWrapper docWrapper = DocumentProtos.DocumentWrapper.parseFrom(value.copyBytes());

    log.info(
        "work title = "
            + docWrapper.getDocumentMetadata().getBasicMetadata().getTitle(0).getText());

    if ((i % 101) > 100 - percentOfWritten) {
      log.info("writing...");
      context.write(new Text(docWrapper.getRowId()), new BytesWritable(value.copyBytes()));
    }

    i++;
  }
コード例 #3
0
    @Override
    public void reduce(
        BytesWritable topkRollupKey, Iterable<BytesWritable> timeSeriesIterable, Context context)
        throws IOException, InterruptedException {

      TopKRollupPhaseOneMapOutputKey wrapper =
          TopKRollupPhaseOneMapOutputKey.fromBytes(topkRollupKey.getBytes());
      LOGGER.info(
          "DimensionName {} DimensionValue {}",
          wrapper.getDimensionName(),
          wrapper.getDimensionValue());

      MetricTimeSeries aggregateSeries = new MetricTimeSeries(metricSchema);
      for (BytesWritable writable : timeSeriesIterable) {
        MetricTimeSeries series = MetricTimeSeries.fromBytes(writable.copyBytes(), metricSchema);
        aggregateSeries.aggregate(series);
      }

      Map<String, Long> metricValues = new HashMap<String, Long>();
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        metricValues.put(metricSpec.getName(), 0L);
      }
      for (Long time : aggregateSeries.getTimeWindowSet()) {
        for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
          String metricName = metricSpec.getName();
          long metricValue = aggregateSeries.get(time, metricName).longValue();
          metricValues.put(metricName, metricValues.get(metricName) + metricValue);
        }
      }

      boolean aboveThreshold = true;
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        String metricName = metricSpec.getName();

        long metricValue = metricValues.get(metricName);
        long metricSum = metricSums.get(metricName);
        double metricThreshold = metricThresholds.get(metricName);

        LOGGER.info("metricValue : {} metricSum : {}", metricValue, metricSum);
        if (metricValue < (metricThreshold / 100) * metricSum) {
          aboveThreshold = false;
          break;
        }
      }

      if (aboveThreshold) {
        LOGGER.info("Passed threshold");
        valWritable.set(aggregateSeries.toBytes(), 0, aggregateSeries.toBytes().length);
        context.write(topkRollupKey, valWritable);
      }
    }
コード例 #4
0
  @Override
  public void map(
      Writable key,
      BytesWritable value,
      Mapper<Writable, BytesWritable, Text, BytesWritable>.Context context)
      throws IOException, InterruptedException {

    DocumentWrapper docWrapper = DocumentProtos.DocumentWrapper.parseFrom(value.copyBytes());

    String docKey = keyGen.generateKey(docWrapper.getDocumentMetadata(), 0);

    if (!docKey.isEmpty()) {
      DocumentWrapper thinDocWrapper = DocumentWrapperUtils.cloneDocumentMetadata(docWrapper);
      context.write(new Text(docKey), new BytesWritable(thinDocWrapper.toByteArray()));
    }
  }
コード例 #5
0
    @Override
    protected void map(Text key, BytesWritable value, Context context)
        throws IOException, InterruptedException {

      StatisticsProtos.InputEntry inputEntry =
          StatisticsProtos.InputEntry.parseFrom(value.copyBytes());

      Set<SortedMapWritableComparable> outputKeyMaps = new HashSet<SortedMapWritableComparable>();
      outputKeyMaps.add(new SortedMapWritableComparable());

      Map<String, Partitioner> partitioners = statGenConfiguration.getPartitioners();

      for (StatisticsProtos.KeyValue field : inputEntry.getFieldList()) {
        String fieldKey = field.getKey();
        if (partitioners.containsKey(fieldKey)) {
          String[] partitions = partitioners.get(fieldKey).partition(field.getValue());
          // the number of output values should be multiplied by number of partitions
          // (partitions.lenght).
          // if 0, then return nothing...
          if (partitions.length == 0) {
            return;
          }
          for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
            outputKeyMap.put(new Text(fieldKey), new Text(partitions[0]));
          }
          // if more than 1, then copy output values
          for (int i = 1; i < partitions.length; i++) {
            Text mapKey = new Text(fieldKey);
            Text mapValue = new Text(partitions[i]);
            Set<SortedMapWritableComparable> newMaps = new HashSet<SortedMapWritableComparable>();
            for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
              SortedMapWritableComparable newMap = new SortedMapWritableComparable(outputKeyMap);
              newMap.put(mapKey, mapValue);
              newMaps.add(newMap);
            }
            outputKeyMaps.addAll(newMaps);
          }
        }
      }

      for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
        context.write(outputKeyMap, value);
      }
    }
コード例 #6
0
 private List<byte[]> readResults(Path outputPath, Configuration config, FileSystem fs)
     throws IOException {
   List<byte[]> ret = new ArrayList<>();
   for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(outputPath, false); it.hasNext(); ) {
     Path p = it.next().getPath();
     if (p.getName().equals("_SUCCESS")) {
       fs.delete(p, false);
       continue;
     }
     SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(p));
     LongWritable key = new LongWritable();
     BytesWritable value = new BytesWritable();
     while (reader.next(key, value)) {
       ret.add(value.copyBytes());
     }
     reader.close();
     fs.delete(p, false);
   }
   fs.delete(outputPath, false);
   if (LOG.isDebugEnabled()) {
     LOG.debug(outputPath + ": Returning " + ret.size());
   }
   return ret;
 }
コード例 #7
0
  @Override
  protected void reduce(Writable key, Iterable<BytesWritable> values, Context context)
      throws IOException, InterruptedException {
    ArrayList<String> docIds = new ArrayList<String>();
    HashSet<String> issns = new HashSet<String>();
    HashSet<String> isbns = new HashSet<String>();
    HashSet<String> titles = new HashSet<String>();
    for (BytesWritable value : values) {

      DocumentProtos.DocumentWrapper docWrapper =
          DocumentProtos.DocumentWrapper.parseFrom(value.copyBytes());
      docIds.add(docWrapper.getRowId());
      String issn = null;
      String isbn = null;
      if (docWrapper.hasDocumentMetadata() && docWrapper.getDocumentMetadata().hasBasicMetadata()) {
        DocumentProtos.BasicMetadataOrBuilder bm =
            docWrapper.getDocumentMetadata().getBasicMetadataOrBuilder();
        if (bm.hasIssn()) {
          issn = bm.getIssn();
        }
        if (bm.hasIsbn()) {
          isbn = bm.getIsbn();
        }
        if (StringUtils.isNotBlank(isbn)) {
          isbn = isbn.trim().toUpperCase();
          isbns.add(isbn);
        }
        if (StringUtils.isNotBlank(issn)) {
          issn = issn.trim().toUpperCase();
          issns.add(issn);
        }
        if (bm.hasJournal()) {
          String title = bm.getJournal().trim();
          if (StringUtils.isNotBlank(title)) {
            titles.add(title);
          }
        }
      }
    }
    String issn = null;
    String isbn = null;
    ArrayList<String> titleA = new ArrayList<String>(titles);
    Collections.sort(
        titleA,
        new Comparator<String>() {
          @Override
          public int compare(String o1, String o2) {
            // najdłuższe naprzód
            return o2.length() - o1.length();
          }
        });

    if (issns.size() > 0) {
      issn = issns.iterator().next();
    }
    if (isbns.size() > 0) {
      isbn = isbns.iterator().next();
    }
    String id = "http://comac.ceon.pl/source-";
    if (issn != null) {
      id += ("issn-" + issn);
      if (isbn != null) {
        id += "-";
      }
    }
    if (isbn != null) {
      id += ("isbn-" + isbn);
    }
    for (String docId : docIds) {
      ParentModelProtos.ParentDisambiguationOut.Builder parent =
          ParentModelProtos.ParentDisambiguationOut.newBuilder();
      parent.setDocId(docId);
      parent.setParentId(id);
      parent.addAllParentName(titles);
      if (isbn != null) {
        parent.setType(DocumentProtos.BasicMetadata.ParentType.BOOK);
      } else {
        parent.setType(DocumentProtos.BasicMetadata.ParentType.JOURNAL);
      }
      context.write(new Text(docId), new BytesWritable(parent.build().toByteArray()));
    }
  }
コード例 #8
0
 @Override
 protected void map(LongWritable key, BytesWritable value, Context context)
     throws IOException, InterruptedException {
   Text output = new Text(value.copyBytes());
   context.write(output, output);
 }
コード例 #9
0
 private Stream<PacketInfo> filteredPacketInfo(BytesWritable value) throws IOException {
   return PcapHelper.toPacketInfo(value.copyBytes()).stream().filter(filter);
 }