コード例 #1
0
  public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
      throws InvalidRequestException, NotFoundException {
    if (logger.isDebugEnabled()) logger.debug("get_column");
    ThriftValidation.validateColumnPath(table, column_path);

    QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
            consistency_level);
    // TODO can we leverage getSlice here and just check that it returns one column?
    if (cfamily == null) {
      throw new NotFoundException();
    }
    Collection<IColumn> columns = null;
    if (column_path.super_column != null) {
      IColumn column = cfamily.getColumn(column_path.super_column);
      if (column != null) {
        columns = column.getSubColumns();
      }
    } else {
      columns = cfamily.getSortedColumns();
    }
    if (columns == null || columns.size() == 0) {
      throw new NotFoundException();
    }

    assert columns.size() == 1;
    IColumn column = columns.iterator().next();
    if (column.isMarkedForDelete()) {
      throw new NotFoundException();
    }

    return new Column(column.name(), column.value(), column.timestamp());
  }
コード例 #2
0
  // Don't playa hate, avronate.
  public GenericArray<Column> avronateSubColumns(Collection<IColumn> columns) {
    if (columns == null || columns.isEmpty()) return EMPTY_SUBCOLUMNS;

    GenericData.Array<Column> avroColumns =
        new GenericData.Array<Column>(columns.size(), Column.SCHEMA$);

    for (IColumn column : columns) {
      if (column.isMarkedForDelete()) continue;

      Column avroColumn = newColumn(column.name(), column.value(), column.timestamp());
      avroColumns.add(avroColumn);
    }

    return avroColumns;
  }
コード例 #3
0
 public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
     throws IOException, InterruptedException {
   IColumn column = columns.get(sourceColumn);
   if (column == null) return;
   String value = ByteBufferUtil.string(column.value());
   String[] list = value.split(",");
   for (String t : list) {
     try {
       Integer.valueOf(t);
       context.write(new Text(t), new IntWritable(1));
     } catch (Exception e) {
       e.printStackTrace();
     }
   }
 }
コード例 #4
0
  private Map<String, ColumnOrSuperColumn> multigetInternal(
      String keyspace, List<String> keys, ColumnPath cp, ConsistencyLevel level)
      throws InvalidRequestException, UnavailableException, TimedOutException {
    AvroValidation.validateColumnPath(keyspace, cp);

    // FIXME: This is repetitive.
    byte[] column, super_column;
    column = cp.column == null ? null : cp.column.array();
    super_column = cp.super_column == null ? null : cp.super_column.array();

    QueryPath path =
        new QueryPath(cp.column_family.toString(), column == null ? null : super_column);
    List<byte[]> nameAsList = Arrays.asList(column == null ? super_column : column);
    List<ReadCommand> commands = new ArrayList<ReadCommand>();
    for (String key : keys) {
      AvroValidation.validateKey(key);
      commands.add(new SliceByNamesReadCommand(keyspace, key, path, nameAsList));
    }

    Map<String, ColumnOrSuperColumn> columnFamiliesMap = new HashMap<String, ColumnOrSuperColumn>();
    Map<String, Collection<IColumn>> columnsMap = multigetColumns(commands, level);

    for (ReadCommand command : commands) {
      ColumnOrSuperColumn columnorsupercolumn;

      Collection<IColumn> columns = columnsMap.get(command.key);
      if (columns == null) {
        columnorsupercolumn = new ColumnOrSuperColumn();
      } else {
        assert columns.size() == 1;
        IColumn col = columns.iterator().next();

        if (col.isMarkedForDelete()) {
          columnorsupercolumn = new ColumnOrSuperColumn();
        } else {
          columnorsupercolumn =
              col instanceof org.apache.cassandra.db.Column
                  ? newColumnOrSuperColumn(newColumn(col.name(), col.value(), col.timestamp()))
                  : newColumnOrSuperColumn(
                      newSuperColumn(col.name(), avronateSubColumns(col.getSubColumns())));
        }
      }
      columnFamiliesMap.put(command.key, columnorsupercolumn);
    }

    return columnFamiliesMap;
  }
コード例 #5
0
ファイル: BriskServer.java プロジェクト: thepaul/brisk
  private LocalOrRemoteBlock getRemoteSubBlock(
      ByteBuffer blockId, ByteBuffer sblockId, int offset, ColumnParent subBlockDataPath)
      throws TimedOutException, UnavailableException, InvalidRequestException, NotFoundException {
    // The column name is the SubBlock id (UUID)
    ReadCommand rc =
        new SliceByNamesReadCommand(
            cfsKeyspace, blockId, subBlockDataPath, Arrays.asList(sblockId));

    try {
      // CL=ONE as there are NOT multiple versions of the blocks.
      List<Row> rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.ONE);

      IColumn col = null;
      try {
        col = validateAndGetColumn(rows, sblockId);
      } catch (NotFoundException e) {
        // This is a best effort to get the value. Sometimes due to the size of
        // the sublocks, the normal replication may time out leaving a replicate without
        // the piece of data. Hence we re try with higher CL.
        rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.QUORUM);
      }

      col = validateAndGetColumn(rows, sblockId);

      ByteBuffer value = col.value();

      if (value.remaining() < offset)
        throw new InvalidRequestException("Invalid offset for block of size: " + value.remaining());

      LocalOrRemoteBlock block = new LocalOrRemoteBlock();
      if (offset > 0) {
        ByteBuffer offsetBlock = value.duplicate();
        offsetBlock.position(offsetBlock.position() + offset);
        block.setRemote_block(offsetBlock);
      } else {
        block.setRemote_block(value);
      }

      return block;
    } catch (IOException e) {
      throw new RuntimeException(e);
    } catch (TimeoutException e) {
      throw new TimedOutException();
    }
  }
コード例 #6
0
  public List<Column> thriftifyColumns(Collection<IColumn> columns, boolean reverseOrder) {
    if (columns == null || columns.isEmpty()) {
      return EMPTY_COLUMNS;
    }

    ArrayList<Column> thriftColumns = new ArrayList<Column>(columns.size());
    for (IColumn column : columns) {
      if (column.isMarkedForDelete()) {
        continue;
      }
      Column thrift_column = new Column(column.name(), column.value(), column.timestamp());
      thriftColumns.add(thrift_column);
    }

    // we have to do the reversing here, since internally we pass results around in ColumnFamily
    // objects, which always sort their columns in the "natural" order
    if (reverseOrder) Collections.reverse(thriftColumns);
    return thriftColumns;
  }
コード例 #7
0
  public Tuple source(Map<String, Object> settings, Object boxedKey, Object boxedColumns)
      throws IOException {
    SortedMap<ByteBuffer, IColumn> columns = (SortedMap<ByteBuffer, IColumn>) boxedColumns;
    ByteBuffer key = (ByteBuffer) boxedKey;

    Tuple result = new Tuple();
    result.add(ByteBufferUtil.string(key));

    Map<String, String> dataTypes = SettingsHelper.getTypes(settings);
    List<String> sourceMappings = SettingsHelper.getSourceMappings(settings);

    Map<String, IColumn> columnsByStringName = new HashMap<String, IColumn>();
    for (ByteBuffer columnName : columns.keySet()) {
      String stringName = ByteBufferUtil.string(columnName);
      logger.debug("column name: {}", stringName);
      IColumn col = columns.get(columnName);
      logger.debug("column: {}", col);
      columnsByStringName.put(stringName, col);
    }

    for (String columnName : sourceMappings) {
      AbstractType columnValueType = SerializerHelper.inferType(dataTypes.get(columnName));
      if (columnValueType != null) {
        try {
          IColumn column = columnsByStringName.get(columnName);
          ByteBuffer serializedVal = column.value();
          Object val = null;
          if (serializedVal != null) {
            val = SerializerHelper.deserialize(serializedVal, columnValueType);
          }
          logger.debug("Putting deserialized column: {}. {}", columnName, val);
          result.add(val);
        } catch (Exception e) {
          throw new RuntimeException("Couldn't deserialize column: " + columnName, e);
        }
      } else {
        throw new RuntimeException("no type given for column: " + columnName);
      }
    }

    return result;
  }
コード例 #8
0
ファイル: ParseMapper.java プロジェクト: jschappet/medline
  @Override
  public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
      throws IOException, InterruptedException {
    context.getCounter(Count.DOCS).increment(1);
    Citation citation = null;
    IColumn column = columns.get(sourceColumn);
    if (column == null) return;
    String value = ByteBufferUtil.string(column.value());
    try {
      citation = new Citation(value, unmarshaller);
    } catch (Exception e) {
      // TODO Auto-generated catch block
      logger.error("Error Message", e);
      // e.printStackTrace();
    }

    if (citation == null) {
      logger.info("Blank Citation: " + ByteBufferUtil.string(key));
    } else {
      ArrayList<Mutation> inserts = new ArrayList<Mutation>();
      inserts.add(getMutation("journalTitle", citation.getJournalTitle()));

      inserts.add(getMutation("abstractText", citation.getAbstractText()));
      inserts.add(getMutation("articleTitle", citation.getArticleTitle()));
      if (citation.getPublished() != null) {
        inserts.add(getMutation("publishDate", "" + citation.getPublished().getTime()));
        inserts.add(getMutation("publishYearMonth", format.format(citation.getPublished())));
      }
      if (citation.getCreated() != null) {
        inserts.add(getMutation("createdDate", "" + citation.getCreated().getTime()));
        inserts.add(getMutation("createdYearMonth", format.format(citation.getCreated())));
      }
      if (citation.getRevised() != null) {
        inserts.add(getMutation("revisedYearMonth", format.format(citation.getRevised())));
      }
      if (citation.getCompleted() != null) {
        inserts.add(getMutation("completedYearMonth", format.format(citation.getCompleted())));
      }
      context.write(key, inserts);
    }
  }
コード例 #9
0
  @Override
  @SuppressWarnings("unchecked")
  public List<OneField> getFields(OneRow onerow) throws Exception {
    System.out.println("GetFields");

    List<OneField> fields = new ArrayList<OneField>();

    ByteBuffer keyBuffer = (ByteBuffer) onerow.getKey();
    SortedMap<ByteBuffer, IColumn> keyValues = (SortedMap<ByteBuffer, IColumn>) onerow.getData();

    for (int i = 0; i < this.inputData.columns(); ++i) {
      ColumnDescriptor column = (ColumnDescriptor) this.inputData.getColumn(i);

      if (column.columnName().equals("recordkey")) {
        OneField oneField = new OneField();
        oneField.type = column.columnTypeCode();
        oneField.val = convertToJavaObject(oneField.type, keyBuffer);
        fields.add(oneField);
      } else {
        IColumn value = keyValues.get(ByteBuffer.wrap(column.columnName().getBytes()));

        if (value != null) {

          OneField oneField = new OneField();
          oneField.type = column.columnTypeCode();
          oneField.val = convertToJavaObject(oneField.type, value.value());
          fields.add(oneField);
        } else {
          OneField oneField = new OneField();
          oneField.type = column.columnTypeCode();
          oneField.val = null;
          fields.add(oneField);
        }
      }
    }

    return fields;
  }
コード例 #10
0
 public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
     throws IOException, InterruptedException {
   IColumn column = columns.get(sourceColumn);
   if (column == null) return;
   String value;
   try {
     value = ByteBufferUtil.string(column.value());
   } catch (Exception e) {
     System.out.println("crash for key : " + ByteBufferUtil.string(key));
     return;
   }
   String[] words = value.split(" ");
   Text ID = new Text(ByteBufferUtil.string(key));
   for (String w : words) {
     if (w.equalsIgnoreCase(keyword)) {
       Text foundWord = new Text(w);
       try {
         context.write(foundWord, ID);
       } catch (InterruptedException e) {
         throw new IOException(e);
       }
     }
   }
 }