示例#1
0
  public SuperColumn get_super_column(
      String table, String key, SuperColumnPath super_column_path, int consistency_level)
      throws InvalidRequestException, NotFoundException {
    if (logger.isDebugEnabled()) logger.debug("get_superColumn");
    ThriftValidation.validateSuperColumnPath(table, super_column_path);

    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(
                table,
                key,
                new QueryPath(super_column_path.column_family),
                Arrays.asList(super_column_path.super_column)),
            consistency_level);
    if (cfamily == null) {
      throw new NotFoundException();
    }
    Collection<IColumn> columns = cfamily.getSortedColumns();
    if (columns == null || columns.size() == 0) {
      throw new NotFoundException();
    }

    assert columns.size() == 1;
    IColumn column = columns.iterator().next();
    if (column.getSubColumns().size() == 0) {
      throw new NotFoundException();
    }

    return new SuperColumn(column.name(), thriftifyColumns(column.getSubColumns()));
  }
示例#2
0
  private Map<String, Collection<IColumn>> multigetColumns(
      List<ReadCommand> commands, ConsistencyLevel level)
      throws InvalidRequestException, UnavailableException, TimedOutException {
    Map<String, ColumnFamily> cfamilies = readColumnFamily(commands, level);
    Map<String, Collection<IColumn>> columnFamiliesMap = new HashMap<String, Collection<IColumn>>();

    for (ReadCommand command : commands) {
      ColumnFamily cfamily = cfamilies.get(command.key);
      if (cfamily == null) continue;

      Collection<IColumn> columns = null;
      if (command.queryPath.superColumnName != null) {
        IColumn column = cfamily.getColumn(command.queryPath.superColumnName);
        if (column != null) {
          columns = column.getSubColumns();
        }
      } else {
        columns = cfamily.getSortedColumns();
      }

      if (columns != null && columns.size() != 0) {
        columnFamiliesMap.put(command.key, columns);
      }
    }

    return columnFamiliesMap;
  }
 /* convenience method */
 public String getColumnsString(Collection<IColumn> columns) {
   StringBuilder builder = new StringBuilder();
   for (IColumn column : columns) {
     builder.append(column.getString(this)).append(",");
   }
   return builder.toString();
 }
示例#4
0
 public static boolean isRelevant(IColumn column, IColumnContainer container, int gcBefore) {
   // the column itself must be not gc-able (it is live, or a still relevant tombstone, or has live
   // subcolumns), (1)
   // and if its container is deleted, the column must be changed more recently than the container
   // tombstone (2)
   // (since otherwise, the only thing repair cares about is the container tombstone)
   long maxChange = column.mostRecentNonGCableChangeAt(gcBefore);
   return (column.getLocalDeletionTime() >= gcBefore
           || maxChange > column.getMarkedForDeleteAt()) // (1)
       && (!container.isMarkedForDelete() || maxChange > container.getMarkedForDeleteAt()); // (2)
 }
示例#5
0
  private IColumn validateAndGetColumn(List<Row> rows, ByteBuffer columnName)
      throws NotFoundException {
    if (rows.isEmpty()) throw new NotFoundException();

    if (rows.size() > 1) throw new RuntimeException("Block id returned more than one row");

    Row row = rows.get(0);
    if (row.cf == null) throw new NotFoundException();

    IColumn col = row.cf.getColumn(columnName);

    if (col == null || !col.isLive()) throw new NotFoundException();

    return col;
  }
示例#6
0
  // Don't playa hate, avronate.
  public GenericArray<Column> avronateSubColumns(Collection<IColumn> columns) {
    if (columns == null || columns.isEmpty()) return EMPTY_SUBCOLUMNS;

    GenericData.Array<Column> avroColumns =
        new GenericData.Array<Column>(columns.size(), Column.SCHEMA$);

    for (IColumn column : columns) {
      if (column.isMarkedForDelete()) continue;

      Column avroColumn = newColumn(column.name(), column.value(), column.timestamp());
      avroColumns.add(avroColumn);
    }

    return avroColumns;
  }
 public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
     throws IOException, InterruptedException {
   IColumn column = columns.get(sourceColumn);
   if (column == null) return;
   String value = ByteBufferUtil.string(column.value());
   String[] list = value.split(",");
   for (String t : list) {
     try {
       Integer.valueOf(t);
       context.write(new Text(t), new IntWritable(1));
     } catch (Exception e) {
       e.printStackTrace();
     }
   }
 }
  @Test
  public void testImportSuperCf() throws IOException, ParseException {
    String jsonUrl = getClass().getClassLoader().getResource("SuperCF.json").getPath();
    File tempSS = tempSSTableFile("Keyspace1", "Super4");
    SSTableImport.importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath());

    // Verify results
    SSTableReader reader =
        SSTableReader.open(tempSS.getPath(), DatabaseDescriptor.getPartitioner());
    QueryFilter qf =
        QueryFilter.getNamesFilter(
            "rowA", new QueryPath("Super4", null, null), "superA".getBytes());
    ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
    IColumn superCol = cf.getColumn("superA".getBytes());
    assert Arrays.equals(
        superCol.getSubColumn("colAA".getBytes()).value(), hexToBytes("76616c75654141"));
  }
示例#9
0
  /** for resultsets of standard columns */
  private List<Column> getSlice(ReadCommand command, int consistency_level)
      throws InvalidRequestException {
    ColumnFamily cfamily = readColumnFamily(command, consistency_level);
    boolean reverseOrder = false;

    if (command instanceof SliceFromReadCommand)
      reverseOrder = !((SliceFromReadCommand) command).isAscending;

    if (cfamily == null || cfamily.getColumnsMap().size() == 0) {
      return EMPTY_COLUMNS;
    }
    if (cfamily.isSuper()) {
      IColumn column = cfamily.getColumnsMap().values().iterator().next();
      return thriftifyColumns(column.getSubColumns(), reverseOrder);
    }
    return thriftifyColumns(cfamily.getSortedColumns(), reverseOrder);
  }
示例#10
0
    private SimpleBlockFetcher() throws IOException {
      IColumnSerializer columnSerializer = emptyColumnFamily.getColumnSerializer();
      int columns = file.readInt();
      for (int i = 0; i < columns; i++) {
        IColumn column = columnSerializer.deserialize(file);
        if (reversed) blockColumns.addFirst(column);
        else blockColumns.addLast(column);

        /* see if we can stop seeking. */
        boolean outOfBounds = false;
        if (!reversed && finishColumn.remaining() > 0)
          outOfBounds = comparator.compare(column.name(), finishColumn) >= 0;
        else if (reversed && startColumn.remaining() > 0)
          outOfBounds = comparator.compare(column.name(), startColumn) >= 0;
        if (outOfBounds) break;
      }
    }
示例#11
0
  private LocalOrRemoteBlock getRemoteSubBlock(
      ByteBuffer blockId, ByteBuffer sblockId, int offset, ColumnParent subBlockDataPath)
      throws TimedOutException, UnavailableException, InvalidRequestException, NotFoundException {
    // The column name is the SubBlock id (UUID)
    ReadCommand rc =
        new SliceByNamesReadCommand(
            cfsKeyspace, blockId, subBlockDataPath, Arrays.asList(sblockId));

    try {
      // CL=ONE as there are NOT multiple versions of the blocks.
      List<Row> rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.ONE);

      IColumn col = null;
      try {
        col = validateAndGetColumn(rows, sblockId);
      } catch (NotFoundException e) {
        // This is a best effort to get the value. Sometimes due to the size of
        // the sublocks, the normal replication may time out leaving a replicate without
        // the piece of data. Hence we re try with higher CL.
        rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.QUORUM);
      }

      col = validateAndGetColumn(rows, sblockId);

      ByteBuffer value = col.value();

      if (value.remaining() < offset)
        throw new InvalidRequestException("Invalid offset for block of size: " + value.remaining());

      LocalOrRemoteBlock block = new LocalOrRemoteBlock();
      if (offset > 0) {
        ByteBuffer offsetBlock = value.duplicate();
        offsetBlock.position(offsetBlock.position() + offset);
        block.setRemote_block(offsetBlock);
      } else {
        block.setRemote_block(value);
      }

      return block;
    } catch (IOException e) {
      throw new RuntimeException(e);
    } catch (TimeoutException e) {
      throw new TimedOutException();
    }
  }
示例#12
0
    public boolean getNextBlock() throws IOException {
      if (curRangeIndex < 0 || curRangeIndex >= indexes.size()) return false;

      /* seek to the correct offset to the data, and calculate the data size */
      IndexHelper.IndexInfo curColPosition = indexes.get(curRangeIndex);

      /* see if this read is really necessary. */
      if (reversed) {
        if ((finishColumn.remaining() > 0
                && comparator.compare(finishColumn, curColPosition.lastName) > 0)
            || (startColumn.remaining() > 0
                && comparator.compare(startColumn, curColPosition.firstName) < 0)) return false;
      } else {
        if ((startColumn.remaining() > 0
                && comparator.compare(startColumn, curColPosition.lastName) > 0)
            || (finishColumn.remaining() > 0
                && comparator.compare(finishColumn, curColPosition.firstName) < 0)) return false;
      }

      boolean outOfBounds = false;
      long positionToSeek = basePosition + curColPosition.offset;

      // With new promoted indexes, our first seek in the data file will happen at that point.
      if (file == null)
        file = originalInput == null ? sstable.getFileDataInput(positionToSeek) : originalInput;

      IColumnSerializer columnSerializer = emptyColumnFamily.getColumnSerializer();
      file.seek(positionToSeek);
      FileMark mark = file.mark();
      while (file.bytesPastMark(mark) < curColPosition.width && !outOfBounds) {
        IColumn column = columnSerializer.deserialize(file);
        if (reversed) blockColumns.addFirst(column);
        else blockColumns.addLast(column);

        /* see if we can stop seeking. */
        if (!reversed && finishColumn.remaining() > 0)
          outOfBounds = comparator.compare(column.name(), finishColumn) >= 0;
        else if (reversed && startColumn.remaining() > 0)
          outOfBounds = comparator.compare(column.name(), startColumn) >= 0;
      }

      if (reversed) curRangeIndex--;
      else curRangeIndex++;
      return true;
    }
示例#13
0
  public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
      throws InvalidRequestException, NotFoundException {
    if (logger.isDebugEnabled()) logger.debug("get_column");
    ThriftValidation.validateColumnPath(table, column_path);

    QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
            consistency_level);
    // TODO can we leverage getSlice here and just check that it returns one column?
    if (cfamily == null) {
      throw new NotFoundException();
    }
    Collection<IColumn> columns = null;
    if (column_path.super_column != null) {
      IColumn column = cfamily.getColumn(column_path.super_column);
      if (column != null) {
        columns = column.getSubColumns();
      }
    } else {
      columns = cfamily.getSortedColumns();
    }
    if (columns == null || columns.size() == 0) {
      throw new NotFoundException();
    }

    assert columns.size() == 1;
    IColumn column = columns.iterator().next();
    if (column.isMarkedForDelete()) {
      throw new NotFoundException();
    }

    return new Column(column.name(), column.value(), column.timestamp());
  }
示例#14
0
  private List<SuperColumn> thriftifySuperColumns(
      Collection<IColumn> columns, boolean reverseOrder) {
    if (columns == null || columns.isEmpty()) {
      return EMPTY_SUPERCOLUMNS;
    }

    ArrayList<SuperColumn> thriftSuperColumns = new ArrayList<SuperColumn>(columns.size());
    for (IColumn column : columns) {
      List<Column> subcolumns = thriftifyColumns(column.getSubColumns());
      if (subcolumns.isEmpty()) {
        continue;
      }
      thriftSuperColumns.add(new SuperColumn(column.name(), subcolumns));
    }

    if (reverseOrder) Collections.reverse(thriftSuperColumns);

    return thriftSuperColumns;
  }
示例#15
0
  public List<Column> thriftifyColumns(Collection<IColumn> columns, boolean reverseOrder) {
    if (columns == null || columns.isEmpty()) {
      return EMPTY_COLUMNS;
    }

    ArrayList<Column> thriftColumns = new ArrayList<Column>(columns.size());
    for (IColumn column : columns) {
      if (column.isMarkedForDelete()) {
        continue;
      }
      Column thrift_column = new Column(column.name(), column.value(), column.timestamp());
      thriftColumns.add(thrift_column);
    }

    // we have to do the reversing here, since internally we pass results around in ColumnFamily
    // objects, which always sort their columns in the "natural" order
    if (reverseOrder) Collections.reverse(thriftColumns);
    return thriftColumns;
  }
  public Tuple source(Map<String, Object> settings, Object boxedKey, Object boxedColumns)
      throws IOException {
    SortedMap<ByteBuffer, IColumn> columns = (SortedMap<ByteBuffer, IColumn>) boxedColumns;
    ByteBuffer key = (ByteBuffer) boxedKey;

    Tuple result = new Tuple();
    result.add(ByteBufferUtil.string(key));

    Map<String, String> dataTypes = SettingsHelper.getTypes(settings);
    List<String> sourceMappings = SettingsHelper.getSourceMappings(settings);

    Map<String, IColumn> columnsByStringName = new HashMap<String, IColumn>();
    for (ByteBuffer columnName : columns.keySet()) {
      String stringName = ByteBufferUtil.string(columnName);
      logger.debug("column name: {}", stringName);
      IColumn col = columns.get(columnName);
      logger.debug("column: {}", col);
      columnsByStringName.put(stringName, col);
    }

    for (String columnName : sourceMappings) {
      AbstractType columnValueType = SerializerHelper.inferType(dataTypes.get(columnName));
      if (columnValueType != null) {
        try {
          IColumn column = columnsByStringName.get(columnName);
          ByteBuffer serializedVal = column.value();
          Object val = null;
          if (serializedVal != null) {
            val = SerializerHelper.deserialize(serializedVal, columnValueType);
          }
          logger.debug("Putting deserialized column: {}. {}", columnName, val);
          result.add(val);
        } catch (Exception e) {
          throw new RuntimeException("Couldn't deserialize column: " + columnName, e);
        }
      } else {
        throw new RuntimeException("no type given for column: " + columnName);
      }
    }

    return result;
  }
示例#17
0
  public int get_column_count(
      String table, String key, ColumnParent column_parent, int consistency_level)
      throws InvalidRequestException {
    if (logger.isDebugEnabled()) logger.debug("get_column_count");
    // validateColumnParent assumes we require simple columns; g_c_c is the only
    // one of the columnParent-taking apis that can also work at the SC level.
    // so we roll a one-off validator here.
    String cfType = ThriftValidation.validateColumnFamily(table, column_parent.column_family);
    if (cfType.equals("Standard") && column_parent.super_column != null) {
      throw new InvalidRequestException(
          "columnfamily alone is required for standard CF " + column_parent.column_family);
    }

    ColumnFamily cfamily;
    cfamily =
        readColumnFamily(
            new SliceFromReadCommand(
                table,
                key,
                column_parent,
                ArrayUtils.EMPTY_BYTE_ARRAY,
                ArrayUtils.EMPTY_BYTE_ARRAY,
                true,
                Integer.MAX_VALUE),
            consistency_level);
    if (cfamily == null) {
      return 0;
    }
    Collection<IColumn> columns = null;
    if (column_parent.super_column != null) {
      IColumn column = cfamily.getColumn(column_parent.super_column);
      if (column != null) {
        columns = column.getSubColumns();
      }
    } else {
      columns = cfamily.getSortedColumns();
    }
    if (columns == null || columns.size() == 0) {
      return 0;
    }
    return columns.size();
  }
示例#18
0
  @Override
  public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
      throws IOException, InterruptedException {
    context.getCounter(Count.DOCS).increment(1);
    Citation citation = null;
    IColumn column = columns.get(sourceColumn);
    if (column == null) return;
    String value = ByteBufferUtil.string(column.value());
    try {
      citation = new Citation(value, unmarshaller);
    } catch (Exception e) {
      // TODO Auto-generated catch block
      logger.error("Error Message", e);
      // e.printStackTrace();
    }

    if (citation == null) {
      logger.info("Blank Citation: " + ByteBufferUtil.string(key));
    } else {
      ArrayList<Mutation> inserts = new ArrayList<Mutation>();
      inserts.add(getMutation("journalTitle", citation.getJournalTitle()));

      inserts.add(getMutation("abstractText", citation.getAbstractText()));
      inserts.add(getMutation("articleTitle", citation.getArticleTitle()));
      if (citation.getPublished() != null) {
        inserts.add(getMutation("publishDate", "" + citation.getPublished().getTime()));
        inserts.add(getMutation("publishYearMonth", format.format(citation.getPublished())));
      }
      if (citation.getCreated() != null) {
        inserts.add(getMutation("createdDate", "" + citation.getCreated().getTime()));
        inserts.add(getMutation("createdYearMonth", format.format(citation.getCreated())));
      }
      if (citation.getRevised() != null) {
        inserts.add(getMutation("revisedYearMonth", format.format(citation.getRevised())));
      }
      if (citation.getCompleted() != null) {
        inserts.add(getMutation("completedYearMonth", format.format(citation.getCompleted())));
      }
      context.write(key, inserts);
    }
  }
  @Override
  @SuppressWarnings("unchecked")
  public List<OneField> getFields(OneRow onerow) throws Exception {
    System.out.println("GetFields");

    List<OneField> fields = new ArrayList<OneField>();

    ByteBuffer keyBuffer = (ByteBuffer) onerow.getKey();
    SortedMap<ByteBuffer, IColumn> keyValues = (SortedMap<ByteBuffer, IColumn>) onerow.getData();

    for (int i = 0; i < this.inputData.columns(); ++i) {
      ColumnDescriptor column = (ColumnDescriptor) this.inputData.getColumn(i);

      if (column.columnName().equals("recordkey")) {
        OneField oneField = new OneField();
        oneField.type = column.columnTypeCode();
        oneField.val = convertToJavaObject(oneField.type, keyBuffer);
        fields.add(oneField);
      } else {
        IColumn value = keyValues.get(ByteBuffer.wrap(column.columnName().getBytes()));

        if (value != null) {

          OneField oneField = new OneField();
          oneField.type = column.columnTypeCode();
          oneField.val = convertToJavaObject(oneField.type, value.value());
          fields.add(oneField);
        } else {
          OneField oneField = new OneField();
          oneField.type = column.columnTypeCode();
          oneField.val = null;
          fields.add(oneField);
        }
      }
    }

    return fields;
  }
 public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
     throws IOException, InterruptedException {
   IColumn column = columns.get(sourceColumn);
   if (column == null) return;
   String value;
   try {
     value = ByteBufferUtil.string(column.value());
   } catch (Exception e) {
     System.out.println("crash for key : " + ByteBufferUtil.string(key));
     return;
   }
   String[] words = value.split(" ");
   Text ID = new Text(ByteBufferUtil.string(key));
   for (String w : words) {
     if (w.equalsIgnoreCase(keyword)) {
       Text foundWord = new Text(w);
       try {
         context.write(foundWord, ID);
       } catch (InterruptedException e) {
         throw new IOException(e);
       }
     }
   }
 }
示例#21
0
  private Map<String, ColumnOrSuperColumn> multigetInternal(
      String keyspace, List<String> keys, ColumnPath cp, ConsistencyLevel level)
      throws InvalidRequestException, UnavailableException, TimedOutException {
    AvroValidation.validateColumnPath(keyspace, cp);

    // FIXME: This is repetitive.
    byte[] column, super_column;
    column = cp.column == null ? null : cp.column.array();
    super_column = cp.super_column == null ? null : cp.super_column.array();

    QueryPath path =
        new QueryPath(cp.column_family.toString(), column == null ? null : super_column);
    List<byte[]> nameAsList = Arrays.asList(column == null ? super_column : column);
    List<ReadCommand> commands = new ArrayList<ReadCommand>();
    for (String key : keys) {
      AvroValidation.validateKey(key);
      commands.add(new SliceByNamesReadCommand(keyspace, key, path, nameAsList));
    }

    Map<String, ColumnOrSuperColumn> columnFamiliesMap = new HashMap<String, ColumnOrSuperColumn>();
    Map<String, Collection<IColumn>> columnsMap = multigetColumns(commands, level);

    for (ReadCommand command : commands) {
      ColumnOrSuperColumn columnorsupercolumn;

      Collection<IColumn> columns = columnsMap.get(command.key);
      if (columns == null) {
        columnorsupercolumn = new ColumnOrSuperColumn();
      } else {
        assert columns.size() == 1;
        IColumn col = columns.iterator().next();

        if (col.isMarkedForDelete()) {
          columnorsupercolumn = new ColumnOrSuperColumn();
        } else {
          columnorsupercolumn =
              col instanceof org.apache.cassandra.db.Column
                  ? newColumnOrSuperColumn(newColumn(col.name(), col.value(), col.timestamp()))
                  : newColumnOrSuperColumn(
                      newSuperColumn(col.name(), avronateSubColumns(col.getSubColumns())));
        }
      }
      columnFamiliesMap.put(command.key, columnorsupercolumn);
    }

    return columnFamiliesMap;
  }
示例#22
0
 private boolean isColumnNeeded(IColumn column) {
   if (startColumn.remaining() == 0 && finishColumn.remaining() == 0) return true;
   else if (startColumn.remaining() == 0 && !reversed)
     return comparator.compare(column.name(), finishColumn) <= 0;
   else if (startColumn.remaining() == 0 && reversed)
     return comparator.compare(column.name(), finishColumn) >= 0;
   else if (finishColumn.remaining() == 0 && !reversed)
     return comparator.compare(column.name(), startColumn) >= 0;
   else if (finishColumn.remaining() == 0 && reversed)
     return comparator.compare(column.name(), startColumn) <= 0;
   else if (!reversed)
     return comparator.compare(column.name(), startColumn) >= 0
         && comparator.compare(column.name(), finishColumn) <= 0;
   else // if reversed
   return comparator.compare(column.name(), startColumn) <= 0
         && comparator.compare(column.name(), finishColumn) >= 0;
 }