public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_column");
ThriftValidation.validateColumnPath(table, column_path);
QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
consistency_level);
// TODO can we leverage getSlice here and just check that it returns one column?
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = null;
if (column_path.super_column != null) {
IColumn column = cfamily.getColumn(column_path.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.isMarkedForDelete()) {
throw new NotFoundException();
}
return new Column(column.name(), column.value(), column.timestamp());
}
// Don't playa hate, avronate.
public GenericArray<Column> avronateSubColumns(Collection<IColumn> columns) {
if (columns == null || columns.isEmpty()) return EMPTY_SUBCOLUMNS;
GenericData.Array<Column> avroColumns =
new GenericData.Array<Column>(columns.size(), Column.SCHEMA$);
for (IColumn column : columns) {
if (column.isMarkedForDelete()) continue;
Column avroColumn = newColumn(column.name(), column.value(), column.timestamp());
avroColumns.add(avroColumn);
}
return avroColumns;
}
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value = ByteBufferUtil.string(column.value());
String[] list = value.split(",");
for (String t : list) {
try {
Integer.valueOf(t);
context.write(new Text(t), new IntWritable(1));
} catch (Exception e) {
e.printStackTrace();
}
}
}
private Map<String, ColumnOrSuperColumn> multigetInternal(
String keyspace, List<String> keys, ColumnPath cp, ConsistencyLevel level)
throws InvalidRequestException, UnavailableException, TimedOutException {
AvroValidation.validateColumnPath(keyspace, cp);
// FIXME: This is repetitive.
byte[] column, super_column;
column = cp.column == null ? null : cp.column.array();
super_column = cp.super_column == null ? null : cp.super_column.array();
QueryPath path =
new QueryPath(cp.column_family.toString(), column == null ? null : super_column);
List<byte[]> nameAsList = Arrays.asList(column == null ? super_column : column);
List<ReadCommand> commands = new ArrayList<ReadCommand>();
for (String key : keys) {
AvroValidation.validateKey(key);
commands.add(new SliceByNamesReadCommand(keyspace, key, path, nameAsList));
}
Map<String, ColumnOrSuperColumn> columnFamiliesMap = new HashMap<String, ColumnOrSuperColumn>();
Map<String, Collection<IColumn>> columnsMap = multigetColumns(commands, level);
for (ReadCommand command : commands) {
ColumnOrSuperColumn columnorsupercolumn;
Collection<IColumn> columns = columnsMap.get(command.key);
if (columns == null) {
columnorsupercolumn = new ColumnOrSuperColumn();
} else {
assert columns.size() == 1;
IColumn col = columns.iterator().next();
if (col.isMarkedForDelete()) {
columnorsupercolumn = new ColumnOrSuperColumn();
} else {
columnorsupercolumn =
col instanceof org.apache.cassandra.db.Column
? newColumnOrSuperColumn(newColumn(col.name(), col.value(), col.timestamp()))
: newColumnOrSuperColumn(
newSuperColumn(col.name(), avronateSubColumns(col.getSubColumns())));
}
}
columnFamiliesMap.put(command.key, columnorsupercolumn);
}
return columnFamiliesMap;
}
private LocalOrRemoteBlock getRemoteSubBlock(
ByteBuffer blockId, ByteBuffer sblockId, int offset, ColumnParent subBlockDataPath)
throws TimedOutException, UnavailableException, InvalidRequestException, NotFoundException {
// The column name is the SubBlock id (UUID)
ReadCommand rc =
new SliceByNamesReadCommand(
cfsKeyspace, blockId, subBlockDataPath, Arrays.asList(sblockId));
try {
// CL=ONE as there are NOT multiple versions of the blocks.
List<Row> rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.ONE);
IColumn col = null;
try {
col = validateAndGetColumn(rows, sblockId);
} catch (NotFoundException e) {
// This is a best effort to get the value. Sometimes due to the size of
// the sublocks, the normal replication may time out leaving a replicate without
// the piece of data. Hence we re try with higher CL.
rows = StorageProxy.read(Arrays.asList(rc), ConsistencyLevel.QUORUM);
}
col = validateAndGetColumn(rows, sblockId);
ByteBuffer value = col.value();
if (value.remaining() < offset)
throw new InvalidRequestException("Invalid offset for block of size: " + value.remaining());
LocalOrRemoteBlock block = new LocalOrRemoteBlock();
if (offset > 0) {
ByteBuffer offsetBlock = value.duplicate();
offsetBlock.position(offsetBlock.position() + offset);
block.setRemote_block(offsetBlock);
} else {
block.setRemote_block(value);
}
return block;
} catch (IOException e) {
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new TimedOutException();
}
}
public List<Column> thriftifyColumns(Collection<IColumn> columns, boolean reverseOrder) {
if (columns == null || columns.isEmpty()) {
return EMPTY_COLUMNS;
}
ArrayList<Column> thriftColumns = new ArrayList<Column>(columns.size());
for (IColumn column : columns) {
if (column.isMarkedForDelete()) {
continue;
}
Column thrift_column = new Column(column.name(), column.value(), column.timestamp());
thriftColumns.add(thrift_column);
}
// we have to do the reversing here, since internally we pass results around in ColumnFamily
// objects, which always sort their columns in the "natural" order
if (reverseOrder) Collections.reverse(thriftColumns);
return thriftColumns;
}
public Tuple source(Map<String, Object> settings, Object boxedKey, Object boxedColumns)
throws IOException {
SortedMap<ByteBuffer, IColumn> columns = (SortedMap<ByteBuffer, IColumn>) boxedColumns;
ByteBuffer key = (ByteBuffer) boxedKey;
Tuple result = new Tuple();
result.add(ByteBufferUtil.string(key));
Map<String, String> dataTypes = SettingsHelper.getTypes(settings);
List<String> sourceMappings = SettingsHelper.getSourceMappings(settings);
Map<String, IColumn> columnsByStringName = new HashMap<String, IColumn>();
for (ByteBuffer columnName : columns.keySet()) {
String stringName = ByteBufferUtil.string(columnName);
logger.debug("column name: {}", stringName);
IColumn col = columns.get(columnName);
logger.debug("column: {}", col);
columnsByStringName.put(stringName, col);
}
for (String columnName : sourceMappings) {
AbstractType columnValueType = SerializerHelper.inferType(dataTypes.get(columnName));
if (columnValueType != null) {
try {
IColumn column = columnsByStringName.get(columnName);
ByteBuffer serializedVal = column.value();
Object val = null;
if (serializedVal != null) {
val = SerializerHelper.deserialize(serializedVal, columnValueType);
}
logger.debug("Putting deserialized column: {}. {}", columnName, val);
result.add(val);
} catch (Exception e) {
throw new RuntimeException("Couldn't deserialize column: " + columnName, e);
}
} else {
throw new RuntimeException("no type given for column: " + columnName);
}
}
return result;
}
@Override
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
context.getCounter(Count.DOCS).increment(1);
Citation citation = null;
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value = ByteBufferUtil.string(column.value());
try {
citation = new Citation(value, unmarshaller);
} catch (Exception e) {
// TODO Auto-generated catch block
logger.error("Error Message", e);
// e.printStackTrace();
}
if (citation == null) {
logger.info("Blank Citation: " + ByteBufferUtil.string(key));
} else {
ArrayList<Mutation> inserts = new ArrayList<Mutation>();
inserts.add(getMutation("journalTitle", citation.getJournalTitle()));
inserts.add(getMutation("abstractText", citation.getAbstractText()));
inserts.add(getMutation("articleTitle", citation.getArticleTitle()));
if (citation.getPublished() != null) {
inserts.add(getMutation("publishDate", "" + citation.getPublished().getTime()));
inserts.add(getMutation("publishYearMonth", format.format(citation.getPublished())));
}
if (citation.getCreated() != null) {
inserts.add(getMutation("createdDate", "" + citation.getCreated().getTime()));
inserts.add(getMutation("createdYearMonth", format.format(citation.getCreated())));
}
if (citation.getRevised() != null) {
inserts.add(getMutation("revisedYearMonth", format.format(citation.getRevised())));
}
if (citation.getCompleted() != null) {
inserts.add(getMutation("completedYearMonth", format.format(citation.getCompleted())));
}
context.write(key, inserts);
}
}
@Override
@SuppressWarnings("unchecked")
public List<OneField> getFields(OneRow onerow) throws Exception {
System.out.println("GetFields");
List<OneField> fields = new ArrayList<OneField>();
ByteBuffer keyBuffer = (ByteBuffer) onerow.getKey();
SortedMap<ByteBuffer, IColumn> keyValues = (SortedMap<ByteBuffer, IColumn>) onerow.getData();
for (int i = 0; i < this.inputData.columns(); ++i) {
ColumnDescriptor column = (ColumnDescriptor) this.inputData.getColumn(i);
if (column.columnName().equals("recordkey")) {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = convertToJavaObject(oneField.type, keyBuffer);
fields.add(oneField);
} else {
IColumn value = keyValues.get(ByteBuffer.wrap(column.columnName().getBytes()));
if (value != null) {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = convertToJavaObject(oneField.type, value.value());
fields.add(oneField);
} else {
OneField oneField = new OneField();
oneField.type = column.columnTypeCode();
oneField.val = null;
fields.add(oneField);
}
}
}
return fields;
}
public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context)
throws IOException, InterruptedException {
IColumn column = columns.get(sourceColumn);
if (column == null) return;
String value;
try {
value = ByteBufferUtil.string(column.value());
} catch (Exception e) {
System.out.println("crash for key : " + ByteBufferUtil.string(key));
return;
}
String[] words = value.split(" ");
Text ID = new Text(ByteBufferUtil.string(key));
for (String w : words) {
if (w.equalsIgnoreCase(keyword)) {
Text foundWord = new Text(w);
try {
context.write(foundWord, ID);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
}
}
