コード例 #1
0
  @Override
  public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException {
    byte[] bytesToWriteIn = bytesWritable.get();
    int offset = bytesWritable.getOffset();

    if (o == null) {
      if (fixedPrefixLength > 0)
        throw new IllegalStateException(
            "excepted at least " + fixedPrefixLength + " bytes to write");
      else if (terminate()) {
        // write one (masked) null byte
        bytesToWriteIn[offset] = mask(NULL);
        RowKeyUtils.seek(bytesWritable, 1);
      }
    } else {
      final BytesWritable input = (BytesWritable) o;
      if (fixedPrefixLength > input.getLength())
        throw new IllegalStateException(
            "excepted at least " + fixedPrefixLength + " bytes to write");
      else {
        encodeFixedPrefix(input.getBytes(), bytesWritable);
        encodedCustomizedReversedPackedBcd(
            toStringRepresentation(
                input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength),
            bytesWritable);
      }
    }
  }
コード例 #2
0
    @Override
    public void map(ImmutableHexWritable key, FsEntry value, Context context)
        throws IOException, InterruptedException {
      if (Extensions.contains(value.extension())) {
        FullPath.set(value.fullPath());
        Ext.set(value.extension());

        encodeHex(Sha, value, "sha1");
        encodeHex(Md5, value, "md5");

        if (value.isContentHDFS()) {
          Vid.setSize(0);
          HdfsPath.set(value.getContentHdfsPath());
        } else {
          final byte[] buf = value.getContentBuffer();
          if (buf == null) {
            LOG.warn(value.fullPath() + " didn't have a content buffer, skipping.");
            return;
          }
          Vid.set(buf, 0, buf.length);
          HdfsPath.set("");
        }
        byte[] keybytes = key.get();
        OutKey.set(keybytes, 0, keybytes.length);
        context.write(OutKey, Fields);
      }
    }
コード例 #3
0
ファイル: SortValidator.java プロジェクト: hramasamy/evosys
      public void reduce(
          BytesWritable key,
          Iterator<IntWritable> values,
          OutputCollector<BytesWritable, IntWritable> output,
          Reporter reporter)
          throws IOException {
        int ones = 0;
        int twos = 0;
        while (values.hasNext()) {
          IntWritable count = values.next();
          if (count.equals(sortInput)) {
            ++ones;
          } else if (count.equals(sortOutput)) {
            ++twos;
          } else {
            throw new IOException(
                "Invalid 'value' of " + count.get() + " for (key,value): " + key.toString());
          }
        }

        // Check to ensure there are equal no. of ones and twos
        if (ones != twos) {
          throw new IOException(
              "Illegal ('one', 'two'): ("
                  + ones
                  + ", "
                  + twos
                  + ") for (key, value): "
                  + key.toString());
        }
      }
コード例 #4
0
 public KeyAndPartitionWritable(Domain domain, BytesWritable key) {
   this.key = key;
   int partition =
       domain
           .getPartitioner()
           .partition(ByteBuffer.wrap(key.getBytes(), 0, key.getLength()), domain.getNumParts());
   this.partition = new IntWritable(partition);
 }
コード例 #5
0
 @Override
 public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
   BytesWritable writable = new BytesWritable();
   ByteBuffer bb = (ByteBuffer) value;
   writable.set(bb.array(), 0, bb.array().length);
   size.value = bb.array().length;
   return writable;
 }
コード例 #6
0
 @Override
 public boolean next(BytesWritable k, BytesWritable v) throws IOException {
   if (nextKeyValue()) {
     k.set(new BytesWritable(getCurrentKey().getBytes()));
     v.set(new BytesWritable(getCurrentValue()));
     return true;
   }
   return false;
 }
コード例 #7
0
 @Override
 public synchronized void write(BytesWritable key, BytesWritable value) throws IOException {
   try {
     mWriter.put(key.getBytes(), value.getBytes());
     // Send a progress to the job manager to inform it that the task is still running.
     mProgress.progress();
   } catch (TachyonException e) {
     throw new IOException(e);
   }
 }
コード例 #8
0
  @Override
  public int getSerializedLength(Object o) throws IOException {
    if (o == null) return terminate() ? fixedPrefixLength + 1 : fixedPrefixLength;

    final BytesWritable input = (BytesWritable) o;
    return fixedPrefixLength
        + getSerializedLength(
            toStringRepresentation(
                input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength));
  }
コード例 #9
0
  @Override
  @SuppressWarnings("unchecked")
  public void run(final JobConf job, final TaskUmbilicalProtocol umbilical) throws IOException {

    final Reporter reporter = getReporter(umbilical);

    // start thread that will handle communication with parent
    startCommunicationThread(umbilical);

    int numReduceTasks = conf.getNumReduceTasks();
    LOG.info("numReduceTasks: " + numReduceTasks);
    MapOutputCollector collector = null;
    if (numReduceTasks > 0) {
      collector = new MapOutputBuffer(umbilical, job, reporter);
    } else {
      collector = new DirectMapOutputCollector(umbilical, job, reporter);
    }
    // reinstantiate the split
    try {
      instantiatedSplit =
          (InputSplit) ReflectionUtils.newInstance(job.getClassByName(splitClass), job);
    } catch (ClassNotFoundException exp) {
      IOException wrap = new IOException("Split class " + splitClass + " not found");
      wrap.initCause(exp);
      throw wrap;
    }
    DataInputBuffer splitBuffer = new DataInputBuffer();
    splitBuffer.reset(split.get(), 0, split.getSize());
    instantiatedSplit.readFields(splitBuffer);

    // if it is a file split, we can give more details
    if (instantiatedSplit instanceof FileSplit) {
      FileSplit fileSplit = (FileSplit) instantiatedSplit;
      job.set("map.input.file", fileSplit.getPath().toString());
      job.setLong("map.input.start", fileSplit.getStart());
      job.setLong("map.input.length", fileSplit.getLength());
    }

    RecordReader rawIn = // open input
        job.getInputFormat().getRecordReader(instantiatedSplit, job, reporter);
    RecordReader in = new TrackedRecordReader(rawIn, getCounters());

    MapRunnable runner = (MapRunnable) ReflectionUtils.newInstance(job.getMapRunnerClass(), job);

    try {
      runner.run(in, collector, reporter);
      collector.flush();
    } finally {
      // close
      in.close(); // close input
      collector.close();
    }
    done(umbilical);
  }
コード例 #10
0
  static Rename[] readRenameOptions(DataInputStream in) throws IOException {
    BytesWritable writable = new BytesWritable();
    writable.readFields(in);

    byte[] bytes = writable.getBytes();
    Rename[] options = new Rename[bytes.length];

    for (int i = 0; i < bytes.length; i++) {
      options[i] = Rename.valueOf(bytes[i]);
    }
    return options;
  }
コード例 #11
0
ファイル: UDFSha1.java プロジェクト: honglvlan/BDST
  /** Convert bytes to SHA-1 */
  public Text evaluate(BytesWritable b) {
    if (b == null) {
      return null;
    }

    digest.reset();
    digest.update(b.getBytes(), 0, b.getLength());
    byte[] shaBytes = digest.digest();
    String shaHex = Hex.encodeHexString(shaBytes);

    result.set(shaHex);
    return result;
  }
コード例 #12
0
    @Override
    public void reduce(
        BytesWritable topkRollupKey, Iterable<BytesWritable> timeSeriesIterable, Context context)
        throws IOException, InterruptedException {

      TopKRollupPhaseOneMapOutputKey wrapper =
          TopKRollupPhaseOneMapOutputKey.fromBytes(topkRollupKey.getBytes());
      LOGGER.info(
          "DimensionName {} DimensionValue {}",
          wrapper.getDimensionName(),
          wrapper.getDimensionValue());

      MetricTimeSeries aggregateSeries = new MetricTimeSeries(metricSchema);
      for (BytesWritable writable : timeSeriesIterable) {
        MetricTimeSeries series = MetricTimeSeries.fromBytes(writable.copyBytes(), metricSchema);
        aggregateSeries.aggregate(series);
      }

      Map<String, Long> metricValues = new HashMap<String, Long>();
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        metricValues.put(metricSpec.getName(), 0L);
      }
      for (Long time : aggregateSeries.getTimeWindowSet()) {
        for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
          String metricName = metricSpec.getName();
          long metricValue = aggregateSeries.get(time, metricName).longValue();
          metricValues.put(metricName, metricValues.get(metricName) + metricValue);
        }
      }

      boolean aboveThreshold = true;
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        String metricName = metricSpec.getName();

        long metricValue = metricValues.get(metricName);
        long metricSum = metricSums.get(metricName);
        double metricThreshold = metricThresholds.get(metricName);

        LOGGER.info("metricValue : {} metricSum : {}", metricValue, metricSum);
        if (metricValue < (metricThreshold / 100) * metricSum) {
          aboveThreshold = false;
          break;
        }
      }

      if (aboveThreshold) {
        LOGGER.info("Passed threshold");
        valWritable.set(aggregateSeries.toBytes(), 0, aggregateSeries.toBytes().length);
        context.write(topkRollupKey, valWritable);
      }
    }
コード例 #13
0
 private Object deserializeValue(BytesWritable valueWritable, byte tag) throws HiveException {
   try {
     return inputValueDeserializer[tag].deserialize(valueWritable);
   } catch (SerDeException e) {
     throw new HiveException(
         "Error: Unable to deserialize reduce input value (tag="
             + tag
             + ") from "
             + Utilities.formatBinaryString(valueWritable.getBytes(), 0, valueWritable.getLength())
             + " with properties "
             + valueTableDesc[tag].getProperties(),
         e);
   }
 }
コード例 #14
0
  /* decrypt, then compute
   */
  @Override
  public void map(BytesWritable key, BytesWritable val, Context context)
      throws IOException, InterruptedException {
    byte[] iv = key.copyBytes();
    byte[] ct = val.copyBytes();
    // String[] ss = new String(crypto.decrypt_word_rnd(ct, iv)).split("\\s+");

    String text = new String(crypto.decrypt_word_rnd(ct, iv));
    Matcher matcher = pattern.matcher(text);

    while (matcher.find()) {
      context.write(new Text(matcher.group(0)), new LongWritable(1));
    }
  }
コード例 #15
0
 @Override
 protected void map(
     final Text key,
     BytesWritable value,
     final Mapper<Text, BytesWritable, Text, BytesWritable>.Context context)
     throws java.io.IOException, InterruptedException {
   if (wordlist.contains(key.toString())) {
     IOUtils.deserialize(
         value.getBytes(),
         new ReadableListBinary<Object>(new ArrayList<Object>()) {
           @Override
           protected Object readValue(DataInput in) throws IOException {
             WordDFIDF idf = new WordDFIDF();
             idf.readBinary(in);
             try {
               ByteArrayOutputStream baos = new ByteArrayOutputStream();
               IOUtils.writeBinary(baos, idf);
               context.write(key, new BytesWritable(baos.toByteArray()));
             } catch (InterruptedException e) {
               throw new IOException("");
             }
             return NullWritable.get();
           }
         });
   }
 };
コード例 #16
0
    @Override
    public void map(
        final Text key,
        BytesWritable value,
        final Mapper<Text, BytesWritable, Text, LongWritable>.Context context)
        throws InterruptedException {
      try {
        final long[] largest = new long[] {0};
        final boolean[] anyDayOverLimit = new boolean[] {false};
        IOUtils.deserialize(
            value.getBytes(),
            new ReadableListBinary<Object>(new ArrayList<Object>()) {
              @Override
              protected Object readValue(DataInput in) throws IOException {
                WordDFIDF idf = new WordDFIDF();
                idf.readBinary(in);
                if (idf.wf > wordTimeCountThresh) {
                  anyDayOverLimit[0] = true;
                }
                if (largest[0] < idf.Twf) largest[0] = idf.Twf;

                return new Object();
              }
            });
        if (anyDayOverLimit[0]) // One of the days was over the day limit
        context.write(key, new LongWritable(largest[0]));

      } catch (IOException e) {
        System.err.println("Couldnt read word: " + key);
      }
    }
コード例 #17
0
    @Override
    public void write(Writable w) throws IOException {

      // Get input data
      byte[] input;
      int inputLength;
      if (w instanceof Text) {
        input = ((Text) w).getBytes();
        inputLength = ((Text) w).getLength();
      } else {
        assert (w instanceof BytesWritable);
        input = ((BytesWritable) w).get();
        inputLength = ((BytesWritable) w).getSize();
      }

      // Add signature
      byte[] wrapped = new byte[signature.length + inputLength];
      for (int i = 0; i < signature.length; i++) {
        wrapped[i] = signature[i];
      }
      for (int i = 0; i < inputLength; i++) {
        wrapped[i + signature.length] = input[i];
      }

      // Encode
      byte[] output = base64.encode(wrapped);
      bytesWritable.set(output, 0, output.length);

      writer.write(bytesWritable);
    }
コード例 #18
0
 @Override
 public void write(DataOutput out) throws IOException {
   super.write(out);
   Text.writeString(out, splitClass);
   split.write(out);
   split = null;
 }
コード例 #19
0
  private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {

    ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
    DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
    int prevDocID = 0;

    try {
      while (true) {
        int left = WritableUtils.readVInt(dataIn);
        int right = WritableUtils.readVInt(dataIn);

        if (right != 0) {
          posting.add(new PairOfInts(left + prevDocID, right));
          prevDocID += left;
        }
      }
    } catch (EOFException e) {
    } catch (IOException e) {
    }

    try {
      dataIn.close();
    } catch (IOException e) {
    }

    return posting;
  }
コード例 #20
0
  public BytesWritable toBytes() // throws IOException
      {
    sbuffer[0] = (byte) (m_isRC ? 1 : 0);

    sbuffer[1] = (byte) ((m_refID & 0xFF000000) >> 24);
    sbuffer[2] = (byte) ((m_refID & 0x00FF0000) >> 16);
    sbuffer[3] = (byte) ((m_refID & 0x0000FF00) >> 8);
    sbuffer[4] = (byte) ((m_refID & 0x000000FF));

    sbuffer[5] = (byte) ((m_refStart & 0xFF000000) >> 24);
    sbuffer[6] = (byte) ((m_refStart & 0x00FF0000) >> 16);
    sbuffer[7] = (byte) ((m_refStart & 0x0000FF00) >> 8);
    sbuffer[8] = (byte) ((m_refStart & 0x000000FF));

    sbuffer[9] = (byte) ((m_refEnd & 0xFF000000) >> 24);
    sbuffer[10] = (byte) ((m_refEnd & 0x00FF0000) >> 16);
    sbuffer[11] = (byte) ((m_refEnd & 0x0000FF00) >> 8);
    sbuffer[12] = (byte) ((m_refEnd & 0x000000FF));

    sbuffer[13] = (byte) ((m_differences & 0xFF000000) >> 24);
    sbuffer[14] = (byte) ((m_differences & 0x00FF0000) >> 16);
    sbuffer[15] = (byte) ((m_differences & 0x0000FF00) >> 8);
    sbuffer[16] = (byte) ((m_differences & 0x000000FF));

    bytes.set(sbuffer, 0, 17);
    return bytes;
  }
コード例 #21
0
ファイル: EtlRecordReader.java プロジェクト: mate1/camus
  private static byte[] getBytes(BytesWritable val) {
    byte[] buffer = val.getBytes();

    /*
     * FIXME: remove the following part once the below jira is fixed
     * https://issues.apache.org/jira/browse/HADOOP-6298
     */
    long len = val.getLength();
    byte[] bytes = buffer;
    if (len < buffer.length) {
      bytes = new byte[(int) len];
      System.arraycopy(buffer, 0, bytes, 0, (int) len);
    }

    return bytes;
  }
コード例 #22
0
  @Override
  public boolean nextKeyValue() throws IOException, InterruptedException {
    FileSystem fileSystem = FileSystem.get(configuration);

    if (fileSystem.isDirectory(split.getPath())) {
      return false;
    }

    if (fileProcessed) {
      return false;
    }

    int fileLength = (int) split.getLength();
    byte[] result = new byte[fileLength];

    FSDataInputStream inputStream = null;

    try {
      inputStream = fileSystem.open(split.getPath());
      IOUtils.readFully(inputStream, result, 0, fileLength);
      currentValue.set(result, 0, fileLength);
    } finally {
      IOUtils.closeStream(inputStream);
    }
    fileProcessed = true;
    return true;
  }
コード例 #23
0
  @SuppressWarnings("unchecked")
  protected void setValue(long length) throws IOException {
    ByteArrayOutputStream baos;
    if (length > 0) {
      baos = new ByteArrayOutputStream((int) length);
    } else {
      baos = new ByteArrayOutputStream();
    }

    int size;
    while ((size = zipIn.read(buf, 0, buf.length)) != -1) {
      baos.write(buf, 0, size);
    }
    if (value instanceof Text) {
      ((Text) value).set(baos.toString(encoding));
    } else if (value instanceof BytesWritable) {
      if (batchSize > 1) {
        // Copy data since XCC won't do it when Content is created.
        value = (VALUEIN) new BytesWritable();
      }
      ((BytesWritable) value).set(baos.toByteArray(), 0, baos.size());
    } else {
      String error = "Unsupported input value class: " + value.getClass();
      LOG.error(error, new UnsupportedOperationException(error));
      key = null;
    }
    baos.close();
  }
コード例 #24
0
 @Override
 public Writable create(Object value, TypeConverter typeConverter, Holder<Integer> size) {
   InputStream is = null;
   try {
     is = typeConverter.convertTo(InputStream.class, value);
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     IOUtils.copyBytes(is, bos, HdfsConstants.DEFAULT_BUFFERSIZE, false);
     BytesWritable writable = new BytesWritable();
     writable.set(bos.toByteArray(), 0, bos.toByteArray().length);
     size.value = bos.toByteArray().length;
     return writable;
   } catch (IOException ex) {
     throw new RuntimeCamelException(ex);
   } finally {
     IOHelper.close(is);
   }
 }
コード例 #25
0
ファイル: SdbSerDe.java プロジェクト: rinack/SequoiaDB
  @Override
  public Object deserialize(Writable wr) throws SerDeException {
    LOG.debug("Entry SdbSerDe::deserialize");

    if (!(wr instanceof BytesWritable)) {
      throw new SerDeException("Expected BSONWritable, received " + wr.getClass().getName());
    }

    BytesWritable record = (BytesWritable) wr;

    ByteArrayRef bytes = new ByteArrayRef();
    bytes.setData(record.getBytes());

    row.init(bytes, 0, record.getLength());

    return row;
  }
コード例 #26
0
  @Override
  public Object deserialize(Writable blob) throws SerDeException {
    BytesWritable data = (BytesWritable) blob;
    inputByteBuffer.reset(data.getBytes(), 0, data.getLength());

    try {
      for (int i = 0; i < columnNames.size(); i++) {
        row.set(
            i,
            deserialize(inputByteBuffer, columnTypes.get(i), columnSortOrderIsDesc[i], row.get(i)));
      }
    } catch (IOException e) {
      throw new SerDeException(e);
    }

    return row;
  }
コード例 #27
0
 public BytesWritable readBytes(BytesWritable bw) throws IOException {
   byte[] bytes = in.readBytes();
   if (bw == null) {
     bw = new BytesWritable(bytes);
   } else {
     bw.set(bytes, 0, bytes.length);
   }
   return bw;
 }
コード例 #28
0
 @Override
 public ByteArrayRef create(BytesWritable bw) {
   if (null == bw) {
     return null;
   }
   ByteArrayRef ba = new ByteArrayRef();
   ba.setData(bw.getBytes());
   return ba;
 }
コード例 #29
0
ファイル: SimpleGapEncoder.java プロジェクト: alexrenz/lash
 @Override
 public void commit() throws IOException {
   target.setSize(uncommittedLength);
   if (compressGaps) {
     writeGap(gamma + 1);
   } else {
     writeGapUncompressed(gamma + 1);
   }
 }
コード例 #30
0
ファイル: TFileRecordReader.java プロジェクト: ktzoumas/tez
 @Override
 public boolean nextKeyValue() throws IOException, InterruptedException {
   valueBytesWritable.setSize(0);
   if (!scanner.advance()) {
     value = null;
     return false;
   }
   TFile.Reader.Scanner.Entry entry = scanner.entry();
   // populate key, value
   entry.getKey(keyBytesWritable);
   StringBuilder k = new StringBuilder();
   // split path contains the machine name. Create the key as splitPath + realKey
   k.append(splitPath.getName()).append(":").append(new String(keyBytesWritable.getBytes()));
   key.set(k.toString());
   entry.getValue(valueBytesWritable);
   value.set(valueBytesWritable.getBytes());
   return true;
 }