Esempio n. 1
1
  private static void testWriteSequence(int[] sentence, int beginId, int endId, int maxGap)
      throws IOException {
    int partitionId = 1;
    SimpleGapEncoder encoder = new SimpleGapEncoder(maxGap, 3, new BytesWritable());
    encoder.setPartitionId(partitionId);

    // print sentence
    System.out.println();
    System.out.println(
        "Sequence: "
            + Arrays.toString(sentence)
            + " / id range: "
            + beginId
            + "-"
            + endId
            + " / max gap: "
            + maxGap);

    encoder.encode(sentence, 0, sentence.length - 1, beginId, endId, false, true);
    ByteArrayInputStream bin =
        new ByteArrayInputStream(encoder.target().getBytes(), 0, encoder.target().getLength());
    DataInputStream in = new DataInputStream(bin);
    System.out.print("Encoded sequence has " + in.available() + " byte(s): ");
    while (in.available() > 0) {
      long v = WritableUtils.readVInt(in);
      System.out.print(v + " ");
    }
    System.out.println();
  }
Esempio n. 2
0
 @Override
 public void readFields(DataInput input) throws IOException {
   int encodedByteLengthAndBool = WritableUtils.readVInt(input);
   int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
   this.byteValue = new byte[byteLength];
   input.readFully(byteValue, 0, byteLength);
   int sortOrderAndDeterminism = WritableUtils.readVInt(input);
   if (sortOrderAndDeterminism <= 2) {
     // client is on an older version
     this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
     this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
     ;
   } else {
     int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
     this.determinism = Determinism.values()[determinismOrdinal];
     int sortOrderValue =
         sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
     this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
   }
   int typeOrdinal = WritableUtils.readVInt(input);
   if (typeOrdinal < 0) {
     this.type = null;
   } else {
     this.type = PDataType.values()[typeOrdinal];
   }
   if (this.byteValue.length == 0) {
     this.value = null;
   } else {
     this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
   }
 }
  private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {

    ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
    DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
    int prevDocID = 0;

    try {
      while (true) {
        int left = WritableUtils.readVInt(dataIn);
        int right = WritableUtils.readVInt(dataIn);

        if (right != 0) {
          posting.add(new PairOfInts(left + prevDocID, right));
          prevDocID += left;
        }
      }
    } catch (EOFException e) {
    } catch (IOException e) {
    }

    try {
      dataIn.close();
    } catch (IOException e) {
    }

    return posting;
  }
Esempio n. 4
0
 public static TupleProjector deserializeProjectorFromScan(Scan scan) {
   byte[] proj = scan.getAttribute(SCAN_PROJECTOR);
   if (proj == null) {
     return null;
   }
   ByteArrayInputStream stream = new ByteArrayInputStream(proj);
   try {
     DataInputStream input = new DataInputStream(stream);
     KeyValueSchema schema = new KeyValueSchema();
     schema.readFields(input);
     int count = WritableUtils.readVInt(input);
     Expression[] expressions = new Expression[count];
     for (int i = 0; i < count; i++) {
       int ordinal = WritableUtils.readVInt(input);
       expressions[i] = ExpressionType.values()[ordinal].newInstance();
       expressions[i].readFields(input);
     }
     return new TupleProjector(schema, expressions);
   } catch (IOException e) {
     throw new RuntimeException(e);
   } finally {
     try {
       stream.close();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 }
Esempio n. 5
0
 @Override
 public void readFields(DataInput input) throws IOException {
   byte[] tableNameBytes = Bytes.readByteArray(input);
   PName tableName = new PNameImpl(tableNameBytes);
   PTableType tableType = PTableType.values()[WritableUtils.readVInt(input)];
   long sequenceNumber = WritableUtils.readVLong(input);
   long timeStamp = input.readLong();
   byte[] pkNameBytes = Bytes.readByteArray(input);
   String pkName = pkNameBytes.length == 0 ? null : Bytes.toString(pkNameBytes);
   int nColumns = WritableUtils.readVInt(input);
   List<PColumn> columns = Lists.newArrayListWithExpectedSize(nColumns);
   for (int i = 0; i < nColumns; i++) {
     PColumn column = new PColumnImpl();
     column.readFields(input);
     columns.add(column);
   }
   Map<String, byte[][]> guidePosts = new HashMap<String, byte[][]>();
   int size = WritableUtils.readVInt(input);
   for (int i = 0; i < size; i++) {
     String key = WritableUtils.readString(input);
     int valueSize = WritableUtils.readVInt(input);
     byte[][] value = new byte[valueSize][];
     for (int j = 0; j < valueSize; j++) {
       value[j] = Bytes.readByteArray(input);
     }
     guidePosts.put(key, value);
   }
   PTableStats stats = new PTableStatsImpl(guidePosts);
   init(tableName, tableType, timeStamp, sequenceNumber, pkName, columns, stats);
 }
Esempio n. 6
0
 @Override
 public void readFields(DataInput input) throws IOException {
   super.readFields(input);
   toType = PDataType.values()[WritableUtils.readVInt(input)];
   toMod = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
   int byteSize = WritableUtils.readVInt(input);
   this.byteSize = byteSize == -1 ? null : byteSize;
 }
 /** Decode Writable postings and compare by id */
 public int compare(byte[] arg0, int arg1, int arg2, byte[] arg3, int arg4, int arg5) {
   try {
     DataInputStream di1 = new DataInputStream(new ByteArrayInputStream(arg0, arg1, arg2));
     DataInputStream di2 = new DataInputStream(new ByteArrayInputStream(arg3, arg4, arg5));
     return WritableUtils.readVInt(di1) - WritableUtils.readVInt(di2);
   } catch (Exception e) {
     return 0;
   }
 }
Esempio n. 8
0
 @Override
 public void readFields(DataInput in) throws IOException {
   int minNullable = WritableUtils.readVInt(in);
   int nFields = WritableUtils.readVInt(in);
   List<Field> fields = Lists.newArrayListWithExpectedSize(nFields);
   for (int i = 0; i < nFields; i++) {
     Field field = new Field();
     field.readFields(in);
     fields.add(field);
   }
   init(minNullable, fields);
 }
 @Override
 public void readFields(DataInput in) throws IOException {
   id = WritableUtils.readVInt(in);
   expirationDate = WritableUtils.readVLong(in);
   int keyLength = WritableUtils.readVInt(in);
   if (keyLength < 0) {
     secret = null;
   } else {
     byte[] keyBytes = new byte[keyLength];
     in.readFully(keyBytes);
     secret = AuthenticationTokenSecretManager.createSecretKey(keyBytes);
   }
 }
 @Override
 public void readFields(DataInput input) throws IOException {
   // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
   // save space
   int typeAndFlag = WritableUtils.readVInt(input);
   isNullable = (typeAndFlag & 0x01) != 0;
   if ((typeAndFlag & 0x02) != 0) {
     scale = WritableUtils.readVInt(input);
   }
   if ((typeAndFlag & 0x04) != 0) {
     maxLength = WritableUtils.readVInt(input);
   }
   type = PDataType.values()[typeAndFlag >>> 3];
   sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
 }
Esempio n. 11
0
 /** Serializes this object. */
 public void readFields(DataInput in) throws IOException {
   int length = WritableUtils.readVInt(in);
   byte[] bytes = new byte[length];
   in.readFully(bytes, 0, length);
   WikipediaPage.readPage(this, new String(bytes, "UTF-8"));
   language = in.readUTF();
 }
 @Override
 public void readFields(DataInput in) throws IOException {
   clear();
   final int termCount = WritableUtils.readVInt(in);
   for (int i = 0; i < termCount; i++) {
     final String term = Text.readString(in);
     final int freq = WritableUtils.readVInt(in);
     final int bf = WritableUtils.readVInt(in);
     insert(freq, term);
     if (bf == 0) continue;
     final int[] blocks = new int[bf];
     blocks[0] = WritableUtils.readVInt(in) - 1;
     for (int j = 1; j < bf; j++) blocks[j] = WritableUtils.readVInt(in) - blocks[j - 1];
     term_blocks.put(term, new TIntHashSet(blocks));
   }
 }
 private static void rawValueToTextBytes(
     DataOutputBuffer dataBuffer, DataInputBuffer inputBuffer, TextBytes textOut)
     throws IOException {
   inputBuffer.reset(dataBuffer.getData(), dataBuffer.getLength());
   int newLength = WritableUtils.readVInt(inputBuffer);
   textOut.set(inputBuffer.getData(), inputBuffer.getPosition(), newLength);
 }
 private void loadDfs(Path dfStatsPath) throws IOException {
   if (dfs != null) return;
   FSDataInputStream dfStatsInput = fileSys.open(dfStatsPath);
   int l = dfStatsInput.readInt();
   if (l != prefixSet.size()) {
     throw new RuntimeException("df length mismatch: " + l + "\t" + prefixSet.size());
   }
   dfs = new int[l];
   for (int i = 0; i < l; i++) dfs[i] = WritableUtils.readVInt(dfStatsInput);
   dfStatsInput.close();
 }
Esempio n. 15
0
 @Override
 public void readFields(DataInput input) throws IOException {
   // Encode isNullable in sign bit of type ordinal (offset by 1, since ordinal could be 0)
   int typeOrdinal = WritableUtils.readVInt(input);
   if (typeOrdinal < 0) {
     typeOrdinal *= -1;
     this.isNullable = true;
   }
   this.type = PDataType.values()[typeOrdinal - 1];
   this.count = WritableUtils.readVInt(input);
   if (this.count < 0) {
     this.count *= -1;
     this.sortOrder = SortOrder.DESC;
   } else {
     this.sortOrder = SortOrder.ASC;
   }
   if (this.type.isFixedWidth() && this.type.getByteSize() == null) {
     this.byteSize = WritableUtils.readVInt(input);
   }
 }
Esempio n. 16
0
 @Override
 public void readFields(DataInput input) throws IOException {
   byte[] columnNameBytes = Bytes.readByteArray(input);
   PName columnName = PNameFactory.newName(columnNameBytes);
   byte[] familyNameBytes = Bytes.readByteArray(input);
   PName familyName = familyNameBytes.length == 0 ? null : PNameFactory.newName(familyNameBytes);
   // TODO: optimize the reading/writing of this b/c it could likely all fit in a single byte or
   // two
   PDataType dataType = PDataType.values()[WritableUtils.readVInt(input)];
   int maxLength = WritableUtils.readVInt(input);
   int scale = WritableUtils.readVInt(input);
   boolean nullable = input.readBoolean();
   int position = WritableUtils.readVInt(input);
   SortOrder sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
   int arrSize = WritableUtils.readVInt(input);
   init(
       columnName,
       familyName,
       dataType,
       maxLength == NO_MAXLENGTH ? null : maxLength,
       scale == NO_SCALE ? null : scale,
       nullable,
       position,
       sortOrder,
       arrSize == -1 ? null : arrSize);
 }
 private static List<Expression> deserializeExpressions(byte[] b) {
   ByteArrayInputStream stream = new ByteArrayInputStream(b);
   try {
     DataInputStream input = new DataInputStream(stream);
     int size = WritableUtils.readVInt(input);
     List<Expression> selectExpressions = Lists.newArrayListWithExpectedSize(size);
     for (int i = 0; i < size; i++) {
       ExpressionType type = ExpressionType.values()[WritableUtils.readVInt(input)];
       Expression selectExpression = type.newInstance();
       selectExpression.readFields(input);
       selectExpressions.add(selectExpression);
     }
     return selectExpressions;
   } catch (IOException e) {
     throw new RuntimeException(e);
   } finally {
     try {
       stream.close();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 }
Esempio n. 18
0
 @Override
 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
   try {
     final int ret;
     di.reset(b1, s1, l1);
     final int x1 = WritableUtils.readVInt(di);
     di.reset(b2, s2, l2);
     final int x2 = WritableUtils.readVInt(di);
     final int t1 = b1[s1 + x1];
     final int t2 = b2[s2 + x2];
     if (t1 == GridmixKey.REDUCE_SPEC || t2 == GridmixKey.REDUCE_SPEC) {
       ret = t1 - t2;
     } else {
       assert t1 == GridmixKey.DATA;
       assert t2 == GridmixKey.DATA;
       ret = WritableComparator.compareBytes(b1, s1, x1, b2, s2, x2);
     }
     di.reset(reset, 0, 0);
     return ret;
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Esempio n. 19
0
 public static Map<String, List<FaunusEdge>> readFields(
     final DataInput in, final Direction idToRead, final long otherId) throws IOException {
   final Map<String, List<FaunusEdge>> edges = new HashMap<String, List<FaunusEdge>>();
   int edgeTypes = in.readShort();
   for (int i = 0; i < edgeTypes; i++) {
     final String label = in.readUTF();
     final int size = WritableUtils.readVInt(in);
     final List<FaunusEdge> temp = new ArrayList<FaunusEdge>(size);
     for (int j = 0; j < size; j++) {
       final FaunusEdge edge = new FaunusEdge();
       edge.readFieldsCompressed(in, idToRead);
       edge.setLabel(label);
       if (idToRead.equals(Direction.OUT)) edge.inVertex = otherId;
       else edge.outVertex = otherId;
       temp.add(edge);
     }
     edges.put(label, temp);
   }
   return edges;
 }
  @Override
  public void readFields(DataInput in) throws IOException {
    int bufferSize = in.readInt();
    byte[] bytes = new byte[bufferSize]; // TODO usa buffer allocator?
    in.readFully(bytes, 0, bufferSize);
    Buffer buffer = Buffer.wrap(bytes);

    int stringCount = WritableUtils.readVInt(in);
    List<String> strings = new ArrayList<String>(stringCount);
    for (int i = 0; i < stringCount; i++) {
      strings.add(in.readUTF());
    }

    Page newPage = Page.wrap(buffer);
    newPage.setStringReferences(strings);
    if (page != null) {
      page.release();
    }
    page = newPage;
  }
Esempio n. 21
0
  public void readFields(DataInput in) throws IOException {
    this.taskid.readFields(in);
    this.progress = in.readFloat();
    this.state = Text.readString(in);
    this.startTime = in.readLong();
    this.finishTime = in.readLong();

    diagnostics = WritableUtils.readStringArray(in);
    counters = new Counters();
    counters.readFields(in);
    currentStatus = WritableUtils.readEnum(in, TIPStatus.class);
    if (currentStatus == TIPStatus.RUNNING) {
      int num = WritableUtils.readVInt(in);
      for (int i = 0; i < num; i++) {
        TaskAttemptID t = new TaskAttemptID();
        t.readFields(in);
        runningAttempts.add(t);
      }
    } else if (currentStatus == TIPStatus.COMPLETE) {
      successfulAttempt.readFields(in);
    }
  }
  @Override
  public void readFields(DataInput in) throws IOException {
    this.freeEntries = WritableUtils.readVInt(in);
    this.distinct = WritableUtils.readVInt(in);
    this.lowWaterMark = WritableUtils.readVInt(in);
    this.highWaterMark = WritableUtils.readVInt(in);
    this.minLoadFactor = in.readDouble();
    this.maxLoadFactor = in.readDouble();

    this.table = new int[WritableUtils.readVInt(in)];
    for (int i = 0; i < table.length; i++) {
      table[i] = WritableUtils.readVInt(in);
    }

    this.values = new double[WritableUtils.readVInt(in)];
    for (int i = 0; i < values.length; i++) {
      values[i] = in.readDouble();
    }

    this.state = new byte[WritableUtils.readVInt(in)];
    for (int i = 0; i < state.length; i++) {
      state[i] = in.readByte();
    }
  }
Esempio n. 23
0
 @Override
 public void readFields(DataInput input) throws IOException {
   this.returnCode = MutationCode.values()[WritableUtils.readVInt(input)];
   this.mutationTime = input.readLong();
   boolean hasTable = input.readBoolean();
   if (hasTable) {
     this.table = new PTableImpl();
     this.table.readFields(input);
   }
   columnName = Bytes.readByteArray(input);
   if (columnName.length > 0) {
     familyName = Bytes.readByteArray(input);
   }
   boolean hasTablesToDelete = input.readBoolean();
   if (hasTablesToDelete) {
     int count = input.readInt();
     tableNamesToDelete = Lists.newArrayListWithExpectedSize(count);
     for (int i = 0; i < count; i++) {
       byte[] tableName = Bytes.readByteArray(input);
       tableNamesToDelete.add(tableName);
     }
   }
 }
Esempio n. 24
0
  /**
   * Creates a <code>DfTableArray</code> object.
   *
   * @param file collection frequency data file path
   * @param fs FileSystem to read from
   * @throws IOException
   */
  public DfTableArray(Path file, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(file);

    this.numTerms = in.readInt();

    dfs = new int[numTerms];

    for (int i = 0; i < numTerms; i++) {
      int df = WritableUtils.readVInt(in);

      dfs[i] = df;
      if (df > maxDf) {
        maxDf = df;
        maxDfTerm = i + 1;
      }

      if (df == 1) {
        numSingletonTerms++;
      }
    }

    in.close();
  }
Esempio n. 25
0
 public void readFields(DataInput in) throws IOException {
   bytes = WritableUtils.readVLong(in);
   records = WritableUtils.readVLong(in);
   checksum = WritableUtils.readVInt(in);
 }
Esempio n. 26
0
 @Override
 public void readFields(DataInput in) throws IOException {
   // TODO Auto-generated method stub
   empName = WritableUtils.readString(in);
   empID = WritableUtils.readVInt(in);
 }
Esempio n. 27
0
 public void readFields(DataInput in) throws IOException {
   docLength = WritableUtils.readVInt(in);
   weightedTerms = new HMapIFW();
   weightedTerms.readFields(in);
 }
Esempio n. 28
0
 @Override
 public void readFields(DataInput in) throws IOException {
   split = WritableUtils.readVInt(in);
   sequence = WritableUtils.readVLong(in);
 }
Esempio n. 29
0
 @Override
 public void readFields(DataInput in) throws IOException {
   id = WritableUtils.readVInt(in);
   distance = WritableUtils.readVInt(in);
 }
Esempio n. 30
0
 @Override
 public void readFields(DataInput arg0) throws IOException {
   // TODO Auto-generated method stub
   id = WritableUtils.readVInt(arg0);
 }