public SuperColumn get_super_column(
String table, String key, SuperColumnPath super_column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_superColumn");
ThriftValidation.validateSuperColumnPath(table, super_column_path);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(
table,
key,
new QueryPath(super_column_path.column_family),
Arrays.asList(super_column_path.super_column)),
consistency_level);
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = cfamily.getSortedColumns();
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.getSubColumns().size() == 0) {
throw new NotFoundException();
}
return new SuperColumn(column.name(), thriftifyColumns(column.getSubColumns()));
}
private Map<String, Collection<IColumn>> multigetColumns(
List<ReadCommand> commands, ConsistencyLevel level)
throws InvalidRequestException, UnavailableException, TimedOutException {
Map<String, ColumnFamily> cfamilies = readColumnFamily(commands, level);
Map<String, Collection<IColumn>> columnFamiliesMap = new HashMap<String, Collection<IColumn>>();
for (ReadCommand command : commands) {
ColumnFamily cfamily = cfamilies.get(command.key);
if (cfamily == null) continue;
Collection<IColumn> columns = null;
if (command.queryPath.superColumnName != null) {
IColumn column = cfamily.getColumn(command.queryPath.superColumnName);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns != null && columns.size() != 0) {
columnFamiliesMap.put(command.key, columns);
}
}
return columnFamiliesMap;
}
/* convenience method */
public String getColumnsString(Collection<IColumn> columns) {
StringBuilder builder = new StringBuilder();
for (IColumn column : columns) {
builder.append(column.getString(this)).append(",");
}
return builder.toString();
}
public static boolean isRelevant(IColumn column, IColumnContainer container, int gcBefore) {
// the column itself must be not gc-able (it is live, or a still relevant tombstone, or has live
// subcolumns), (1)
// and if its container is deleted, the column must be changed more recently than the container
// tombstone (2)
// (since otherwise, the only thing repair cares about is the container tombstone)
long maxChange = column.mostRecentNonGCableChangeAt(gcBefore);
return (column.getLocalDeletionTime() >= gcBefore
|| maxChange > column.getMarkedForDeleteAt()) // (1)
&& (!container.isMarkedForDelete() || maxChange > container.getMarkedForDeleteAt()); // (2)
}
private IColumn validateAndGetColumn(List<Row> rows, ByteBuffer columnName)
throws NotFoundException {
if (rows.isEmpty()) throw new NotFoundException();
if (rows.size() > 1) throw new RuntimeException("Block id returned more than one row");
Row row = rows.get(0);
if (row.cf == null) throw new NotFoundException();
IColumn col = row.cf.getColumn(columnName);
if (col == null || !col.isLive()) throw new NotFoundException();
return col;
}
// Don't playa hate, avronate.
public GenericArray<Column> avronateSubColumns(Collection<IColumn> columns) {
if (columns == null || columns.isEmpty()) return EMPTY_SUBCOLUMNS;
GenericData.Array<Column> avroColumns =
new GenericData.Array<Column>(columns.size(), Column.SCHEMA$);
for (IColumn column : columns) {
if (column.isMarkedForDelete()) continue;
Column avroColumn = newColumn(column.name(), column.value(), column.timestamp());
avroColumns.add(avroColumn);
}
return avroColumns;
}
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value = ByteBufferUtil.string(column.value());
String[] list = value.split(",");
for (String t : list) {
try {
Integer.valueOf(t);
context.write(new Text(t), new IntWritable(1));
} catch (Exception e) {
e.printStackTrace();
}
}
}
@Test
public void testImportSuperCf() throws IOException, ParseException {
String jsonUrl = getClass().getClassLoader().getResource("SuperCF.json").getPath();
File tempSS = tempSSTableFile("Keyspace1", "Super4");
SSTableImport.importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath());
// Verify results
SSTableReader reader =
SSTableReader.open(tempSS.getPath(), DatabaseDescriptor.getPartitioner());
QueryFilter qf =
QueryFilter.getNamesFilter(
"rowA", new QueryPath("Super4", null, null), "superA".getBytes());
ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily();
IColumn superCol = cf.getColumn("superA".getBytes());
assert Arrays.equals(
superCol.getSubColumn("colAA".getBytes()).value(), hexToBytes("76616c75654141"));
}
/** for resultsets of standard columns */
private List<Column> getSlice(ReadCommand command, int consistency_level)
throws InvalidRequestException {
ColumnFamily cfamily = readColumnFamily(command, consistency_level);
boolean reverseOrder = false;
if (command instanceof SliceFromReadCommand)
reverseOrder = !((SliceFromReadCommand) command).isAscending;
if (cfamily == null || cfamily.getColumnsMap().size() == 0) {
return EMPTY_COLUMNS;
}
if (cfamily.isSuper()) {
IColumn column = cfamily.getColumnsMap().values().iterator().next();
return thriftifyColumns(column.getSubColumns(), reverseOrder);
}
return thriftifyColumns(cfamily.getSortedColumns(), reverseOrder);
}
private SimpleBlockFetcher() throws IOException {
IColumnSerializer columnSerializer = emptyColumnFamily.getColumnSerializer();
int columns = file.readInt();
for (int i = 0; i < columns; i++) {
IColumn column = columnSerializer.deserialize(file);
if (reversed) blockColumns.addFirst(column);
else blockColumns.addLast(column);
/* see if we can stop seeking. */
boolean outOfBounds = false;
if (!reversed && finishColumn.remaining() > 0)
outOfBounds = comparator.compare(column.name(), finishColumn) >= 0;
else if (reversed && startColumn.remaining() > 0)
outOfBounds = comparator.compare(column.name(), startColumn) >= 0;
if (outOfBounds) break;
}
}
private LocalOrRemoteBlock getRemoteSubBlock(
ByteBuffer blockId, ByteBuffer sblockId, int offset, ColumnParent subBlockDataPath)
throws TimedOutException, UnavailableException, InvalidRequestException, NotFoundException {
// The column name is the SubBlock id (UUID)
ReadCommand rc =
new SliceByNamesReadCommand(
cfsKeyspace, blockId, subBlockDataPath, Arrays.asList(sblockId));
try {
// CL=ONE as there are NOT multiple versions of the blocks.
List<Row> rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.ONE);
IColumn col = null;
try {
col = validateAndGetColumn(rows, sblockId);
} catch (NotFoundException e) {
// This is a best effort to get the value. Sometimes due to the size of
// the sublocks, the normal replication may time out leaving a replicate without
// the piece of data. Hence we re try with higher CL.
rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.QUORUM);
}
col = validateAndGetColumn(rows, sblockId);
ByteBuffer value = col.value();
if (value.remaining() < offset)
throw new InvalidRequestException("Invalid offset for block of size: " + value.remaining());
LocalOrRemoteBlock block = new LocalOrRemoteBlock();
if (offset > 0) {
ByteBuffer offsetBlock = value.duplicate();
offsetBlock.position(offsetBlock.position() + offset);
block.setRemote_block(offsetBlock);
} else {
block.setRemote_block(value);
}
return block;
} catch (IOException e) {
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new TimedOutException();
}
}
public boolean getNextBlock() throws IOException {
if (curRangeIndex < 0 || curRangeIndex >= indexes.size()) return false;
/* seek to the correct offset to the data, and calculate the data size */
IndexHelper.IndexInfo curColPosition = indexes.get(curRangeIndex);
/* see if this read is really necessary. */
if (reversed) {
if ((finishColumn.remaining() > 0
&& comparator.compare(finishColumn, curColPosition.lastName) > 0)
|| (startColumn.remaining() > 0
&& comparator.compare(startColumn, curColPosition.firstName) < 0)) return false;
} else {
if ((startColumn.remaining() > 0
&& comparator.compare(startColumn, curColPosition.lastName) > 0)
|| (finishColumn.remaining() > 0
&& comparator.compare(finishColumn, curColPosition.firstName) < 0)) return false;
}
boolean outOfBounds = false;
long positionToSeek = basePosition + curColPosition.offset;
// With new promoted indexes, our first seek in the data file will happen at that point.
if (file == null)
file = originalInput == null ? sstable.getFileDataInput(positionToSeek) : originalInput;
IColumnSerializer columnSerializer = emptyColumnFamily.getColumnSerializer();
file.seek(positionToSeek);
FileMark mark = file.mark();
while (file.bytesPastMark(mark) < curColPosition.width && !outOfBounds) {
IColumn column = columnSerializer.deserialize(file);
if (reversed) blockColumns.addFirst(column);
else blockColumns.addLast(column);
/* see if we can stop seeking. */
if (!reversed && finishColumn.remaining() > 0)
outOfBounds = comparator.compare(column.name(), finishColumn) >= 0;
else if (reversed && startColumn.remaining() > 0)
outOfBounds = comparator.compare(column.name(), startColumn) >= 0;
}
if (reversed) curRangeIndex--;
else curRangeIndex++;
return true;
}
public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_column");
ThriftValidation.validateColumnPath(table, column_path);
QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
consistency_level);
// TODO can we leverage getSlice here and just check that it returns one column?
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = null;
if (column_path.super_column != null) {
IColumn column = cfamily.getColumn(column_path.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.isMarkedForDelete()) {
throw new NotFoundException();
}
return new Column(column.name(), column.value(), column.timestamp());
}
private List<SuperColumn> thriftifySuperColumns(
Collection<IColumn> columns, boolean reverseOrder) {
if (columns == null || columns.isEmpty()) {
return EMPTY_SUPERCOLUMNS;
}
ArrayList<SuperColumn> thriftSuperColumns = new ArrayList<SuperColumn>(columns.size());
for (IColumn column : columns) {
List<Column> subcolumns = thriftifyColumns(column.getSubColumns());
if (subcolumns.isEmpty()) {
continue;
}
thriftSuperColumns.add(new SuperColumn(column.name(), subcolumns));
}
if (reverseOrder) Collections.reverse(thriftSuperColumns);
return thriftSuperColumns;
}
public List<Column> thriftifyColumns(Collection<IColumn> columns, boolean reverseOrder) {
if (columns == null || columns.isEmpty()) {
return EMPTY_COLUMNS;
}
ArrayList<Column> thriftColumns = new ArrayList<Column>(columns.size());
for (IColumn column : columns) {
if (column.isMarkedForDelete()) {
continue;
}
Column thrift_column = new Column(column.name(), column.value(), column.timestamp());
thriftColumns.add(thrift_column);
}
// we have to do the reversing here, since internally we pass results around in ColumnFamily
// objects, which always sort their columns in the "natural" order
if (reverseOrder) Collections.reverse(thriftColumns);
return thriftColumns;
}
public Tuple source(Map<String, Object> settings, Object boxedKey, Object boxedColumns)
throws IOException {
SortedMap<ByteBuffer, IColumn> columns = (SortedMap<ByteBuffer, IColumn>) boxedColumns;
ByteBuffer key = (ByteBuffer) boxedKey;
Tuple result = new Tuple();
result.add(ByteBufferUtil.string(key));
Map<String, String> dataTypes = SettingsHelper.getTypes(settings);
List<String> sourceMappings = SettingsHelper.getSourceMappings(settings);
Map<String, IColumn> columnsByStringName = new HashMap<String, IColumn>();
for (ByteBuffer columnName : columns.keySet()) {
String stringName = ByteBufferUtil.string(columnName);
logger.debug("column name: {}", stringName);
IColumn col = columns.get(columnName);
logger.debug("column: {}", col);
columnsByStringName.put(stringName, col);
}
for (String columnName : sourceMappings) {
AbstractType columnValueType = SerializerHelper.inferType(dataTypes.get(columnName));
if (columnValueType != null) {
try {
IColumn column = columnsByStringName.get(columnName);
ByteBuffer serializedVal = column.value();
Object val = null;
if (serializedVal != null) {
val = SerializerHelper.deserialize(serializedVal, columnValueType);
}
logger.debug("Putting deserialized column: {}. {}", columnName, val);
result.add(val);
} catch (Exception e) {
throw new RuntimeException("Couldn't deserialize column: " + columnName, e);
}
} else {
throw new RuntimeException("no type given for column: " + columnName);
}
}
return result;
}
public int get_column_count(
String table, String key, ColumnParent column_parent, int consistency_level)
throws InvalidRequestException {
if (logger.isDebugEnabled()) logger.debug("get_column_count");
// validateColumnParent assumes we require simple columns; g_c_c is the only
// one of the columnParent-taking apis that can also work at the SC level.
// so we roll a one-off validator here.
String cfType = ThriftValidation.validateColumnFamily(table, column_parent.column_family);
if (cfType.equals("Standard") && column_parent.super_column != null) {
throw new InvalidRequestException(
"columnfamily alone is required for standard CF " + column_parent.column_family);
}
ColumnFamily cfamily;
cfamily =
readColumnFamily(
new SliceFromReadCommand(
table,
key,
column_parent,
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
Integer.MAX_VALUE),
consistency_level);
if (cfamily == null) {
return 0;
}
Collection<IColumn> columns = null;
if (column_parent.super_column != null) {
IColumn column = cfamily.getColumn(column_parent.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
return 0;
}
return columns.size();
}
@Override
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
context.getCounter(Count.DOCS).increment(1);
Citation citation = null;
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value = ByteBufferUtil.string(column.value());
try {
citation = new Citation(value, unmarshaller);
} catch (Exception e) {
// TODO Auto-generated catch block
logger.error("Error Message", e);
// e.printStackTrace();
}
if (citation == null) {
logger.info("Blank Citation: " + ByteBufferUtil.string(key));
} else {
ArrayList<Mutation> inserts = new ArrayList<Mutation>();
inserts.add(getMutation("journalTitle", citation.getJournalTitle()));
inserts.add(getMutation("abstractText", citation.getAbstractText()));
inserts.add(getMutation("articleTitle", citation.getArticleTitle()));
if (citation.getPublished() != null) {
inserts.add(getMutation("publishDate", "" + citation.getPublished().getTime()));
inserts.add(getMutation("publishYearMonth", format.format(citation.getPublished())));
}
if (citation.getCreated() != null) {
inserts.add(getMutation("createdDate", "" + citation.getCreated().getTime()));
inserts.add(getMutation("createdYearMonth", format.format(citation.getCreated())));
}
if (citation.getRevised() != null) {
inserts.add(getMutation("revisedYearMonth", format.format(citation.getRevised())));
}
if (citation.getCompleted() != null) {
inserts.add(getMutation("completedYearMonth", format.format(citation.getCompleted())));
}
context.write(key, inserts);
}
}
@Override
@SuppressWarnings("unchecked")
public List<OneField> getFields(OneRow onerow) throws Exception {
System.out.println("GetFields");
List<OneField> fields = new ArrayList<OneField>();
ByteBuffer keyBuffer = (ByteBuffer) onerow.getKey();
SortedMap<ByteBuffer, IColumn> keyValues = (SortedMap<ByteBuffer, IColumn>) onerow.getData();
for (int i = 0; i < this.inputData.columns(); ++i) {
ColumnDescriptor column = (ColumnDescriptor) this.inputData.getColumn(i);
if (column.columnName().equals("recordkey")) {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = convertToJavaObject(oneField.type, keyBuffer);
fields.add(oneField);
} else {
IColumn value = keyValues.get(ByteBuffer.wrap(column.columnName().getBytes()));
if (value != null) {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = convertToJavaObject(oneField.type, value.value());
fields.add(oneField);
} else {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = null;
fields.add(oneField);
}
}
}
return fields;
}
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value;
try {
value = ByteBufferUtil.string(column.value());
} catch (Exception e) {
System.out.println("crash for key : " + ByteBufferUtil.string(key));
return;
}
String[] words = value.split(" ");
Text ID = new Text(ByteBufferUtil.string(key));
for (String w : words) {
if (w.equalsIgnoreCase(keyword)) {
Text foundWord = new Text(w);
try {
context.write(foundWord, ID);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
}
}
private Map<String, ColumnOrSuperColumn> multigetInternal(
String keyspace, List<String> keys, ColumnPath cp, ConsistencyLevel level)
throws InvalidRequestException, UnavailableException, TimedOutException {
AvroValidation.validateColumnPath(keyspace, cp);
// FIXME: This is repetitive.
byte[] column, super_column;
column = cp.column == null ? null : cp.column.array();
super_column = cp.super_column == null ? null : cp.super_column.array();
QueryPath path =
new QueryPath(cp.column_family.toString(), column == null ? null : super_column);
List<byte[]> nameAsList = Arrays.asList(column == null ? super_column : column);
List<ReadCommand> commands = new ArrayList<ReadCommand>();
for (String key : keys) {
AvroValidation.validateKey(key);
commands.add(new SliceByNamesReadCommand(keyspace, key, path, nameAsList));
}
Map<String, ColumnOrSuperColumn> columnFamiliesMap = new HashMap<String, ColumnOrSuperColumn>();
Map<String, Collection<IColumn>> columnsMap = multigetColumns(commands, level);
for (ReadCommand command : commands) {
ColumnOrSuperColumn columnorsupercolumn;
Collection<IColumn> columns = columnsMap.get(command.key);
if (columns == null) {
columnorsupercolumn = new ColumnOrSuperColumn();
} else {
assert columns.size() == 1;
IColumn col = columns.iterator().next();
if (col.isMarkedForDelete()) {
columnorsupercolumn = new ColumnOrSuperColumn();
} else {
columnorsupercolumn =
col instanceof org.apache.cassandra.db.Column
? newColumnOrSuperColumn(newColumn(col.name(), col.value(), col.timestamp()))
: newColumnOrSuperColumn(
newSuperColumn(col.name(), avronateSubColumns(col.getSubColumns())));
}
}
columnFamiliesMap.put(command.key, columnorsupercolumn);
}
return columnFamiliesMap;
}
private boolean isColumnNeeded(IColumn column) {
if (startColumn.remaining() == 0 && finishColumn.remaining() == 0) return true;
else if (startColumn.remaining() == 0 && !reversed)
return comparator.compare(column.name(), finishColumn) <= 0;
else if (startColumn.remaining() == 0 && reversed)
return comparator.compare(column.name(), finishColumn) >= 0;
else if (finishColumn.remaining() == 0 && !reversed)
return comparator.compare(column.name(), startColumn) >= 0;
else if (finishColumn.remaining() == 0 && reversed)
return comparator.compare(column.name(), startColumn) <= 0;
else if (!reversed)
return comparator.compare(column.name(), startColumn) >= 0
&& comparator.compare(column.name(), finishColumn) <= 0;
else // if reversed
return comparator.compare(column.name(), startColumn) <= 0
&& comparator.compare(column.name(), finishColumn) >= 0;
}