public static DataTable getDataTableFromRowSet(
Collection<Serializable[]> rowEventsSet, DataSchema dataSchema) throws Exception {
final DataTableBuilder dataTableBuilder = new DataTableBuilder(dataSchema);
dataTableBuilder.open();
final Iterator<Serializable[]> iterator = rowEventsSet.iterator();
while (iterator.hasNext()) {
final Serializable[] row = iterator.next();
dataTableBuilder.startRow();
for (int i = 0; i < dataSchema.size(); ++i) {
if (dataSchema.getColumnType(i).isSingleValue()) {
switch (dataSchema.getColumnType(i)) {
case INT:
dataTableBuilder.setColumn(i, ((Integer) row[i]).intValue());
break;
case LONG:
dataTableBuilder.setColumn(i, ((Long) row[i]).longValue());
break;
case DOUBLE:
dataTableBuilder.setColumn(i, ((Double) row[i]).doubleValue());
break;
case FLOAT:
dataTableBuilder.setColumn(i, ((Float) row[i]).floatValue());
break;
case STRING:
dataTableBuilder.setColumn(i, ((String) row[i]));
break;
default:
dataTableBuilder.setColumn(i, row[i]);
break;
}
} else {
switch (dataSchema.getColumnType(i)) {
case INT_ARRAY:
dataTableBuilder.setColumn(i, (int[]) row[i]);
break;
case LONG_ARRAY:
dataTableBuilder.setColumn(i, (long[]) row[i]);
break;
case DOUBLE_ARRAY:
dataTableBuilder.setColumn(i, (double[]) row[i]);
break;
case FLOAT_ARRAY:
dataTableBuilder.setColumn(i, (float[]) row[i]);
break;
case STRING_ARRAY:
dataTableBuilder.setColumn(i, (String[]) row[i]);
break;
default:
dataTableBuilder.setColumn(i, row[i]);
break;
}
}
}
dataTableBuilder.finishRow();
}
dataTableBuilder.seal();
return dataTableBuilder.build();
}
private static JSONArray getSelectionColumnsFromDataSchema(
List<String> selectionColumns, DataSchema dataSchema) {
final JSONArray jsonArray = new JSONArray();
for (int idx = 0; idx < dataSchema.size(); ++idx) {
if (selectionColumns.contains(dataSchema.getColumnName(idx))) {
jsonArray.put(dataSchema.getColumnName(idx));
}
}
return jsonArray;
}
public static List<String> getSelectionColumns(
List<String> selectionColumns, DataSchema dataSchema) {
if ((selectionColumns.size() == 1) && selectionColumns.get(0).equals("*")) {
selectionColumns.clear();
for (int i = 0; i < dataSchema.size(); ++i) {
selectionColumns.add(dataSchema.getColumnName(i));
}
}
return selectionColumns;
}
public static String getRowStringFromSerializable(Serializable[] row, DataSchema dataSchema) {
String rowString = "";
if (dataSchema.getColumnType(0).isSingleValue()) {
if (dataSchema.getColumnType(0) == DataType.STRING) {
rowString += (String) row[0];
} else {
rowString += row[0];
}
} else {
rowString += "[ ";
if (dataSchema.getColumnType(0) == DataType.STRING) {
String[] values = (String[]) row[0];
for (int i = 0; i < values.length; ++i) {
rowString += values[i];
}
} else {
Serializable[] values = (Serializable[]) row[0];
for (int i = 0; i < values.length; ++i) {
rowString += values[i];
}
}
rowString += " ]";
}
for (int i = 1; i < row.length; ++i) {
if (dataSchema.getColumnType(i).isSingleValue()) {
if (dataSchema.getColumnType(i) == DataType.STRING) {
rowString += " : " + (String) row[i];
} else {
rowString += " : " + row[i];
}
} else {
rowString += " : [ ";
switch (dataSchema.getColumnType(i)) {
case STRING_ARRAY:
String[] stringValues = (String[]) row[i];
for (int j = 0; j < stringValues.length; ++j) {
rowString += stringValues[j] + " ";
}
break;
case INT_ARRAY:
int[] intValues = (int[]) row[i];
for (int j = 0; j < intValues.length; ++j) {
rowString += intValues[j] + " ";
}
break;
case FLOAT_ARRAY:
float[] floatValues = (float[]) row[i];
for (int j = 0; j < floatValues.length; ++j) {
rowString += floatValues[j] + " ";
}
break;
case LONG_ARRAY:
long[] longValues = (long[]) row[i];
for (int j = 0; j < longValues.length; ++j) {
rowString += longValues[j] + " ";
}
break;
case DOUBLE_ARRAY:
double[] doubleValues = (double[]) row[i];
for (int j = 0; j < doubleValues.length; ++j) {
rowString += doubleValues[j] + " ";
}
break;
default:
break;
}
rowString += "]";
}
}
return rowString;
}
public static JSONArray getJSonArrayFromRow(
Serializable[] poll, List<String> selectionColumns, DataSchema dataSchema)
throws JSONException {
final JSONArray jsonArray = new JSONArray();
for (int i = 0; i < dataSchema.size(); ++i) {
if (selectionColumns.contains(dataSchema.getColumnName(i))) {
if (dataSchema.getColumnType(i).isSingleValue()) {
if (dataSchema.getColumnType(i) == DataType.STRING) {
jsonArray.put(poll[i]);
} else {
jsonArray.put(
DEFAULT_FORMAT_STRING_MAP.get(dataSchema.getColumnType(i)).format(poll[i]));
}
} else {
// Multi-value;
JSONArray stringJsonArray = new JSONArray();
// stringJsonArray.put(poll[i]);
switch (dataSchema.getColumnType(i)) {
case STRING_ARRAY:
String[] stringValues = (String[]) poll[i];
for (String s : stringValues) {
stringJsonArray.put(s);
}
break;
case INT_ARRAY:
int[] intValues = (int[]) poll[i];
for (int s : intValues) {
stringJsonArray.put(
DEFAULT_FORMAT_STRING_MAP.get(dataSchema.getColumnType(i)).format(s));
}
break;
case FLOAT_ARRAY:
float[] floatValues = (float[]) poll[i];
for (float s : floatValues) {
stringJsonArray.put(
DEFAULT_FORMAT_STRING_MAP.get(dataSchema.getColumnType(i)).format(s));
}
break;
case LONG_ARRAY:
long[] longValues = (long[]) poll[i];
for (long s : longValues) {
stringJsonArray.put(
DEFAULT_FORMAT_STRING_MAP.get(dataSchema.getColumnType(i)).format(s));
}
break;
case DOUBLE_ARRAY:
double[] doubleValues = (double[]) poll[i];
for (double s : doubleValues) {
stringJsonArray.put(
DEFAULT_FORMAT_STRING_MAP.get(dataSchema.getColumnType(i)).format(s));
}
break;
default:
break;
}
jsonArray.put(stringJsonArray);
}
}
}
return jsonArray;
}
public static Serializable[] collectRowFromBlockValSets(
int docId, Block[] blocks, DataSchema dataSchema) {
final Serializable[] row = new Serializable[dataSchema.size()];
int j = 0;
for (int i = 0; i < dataSchema.size(); ++i) {
if (blocks[j] instanceof RealtimeSingleValueBlock) {
if (blocks[j].getMetadata().hasDictionary()) {
Dictionary dictionaryReader = blocks[j].getMetadata().getDictionary();
BlockSingleValIterator bvIter =
(BlockSingleValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
switch (dataSchema.getColumnType(i)) {
case INT:
row[i] = (Integer) ((IntMutableDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case FLOAT:
row[i] = (Float) ((FloatMutableDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case LONG:
row[i] = (Long) ((LongMutableDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case DOUBLE:
row[i] =
(Double) ((DoubleMutableDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case STRING:
row[i] =
(String) ((StringMutableDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
default:
break;
}
} else {
BlockSingleValIterator bvIter =
(BlockSingleValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
switch (dataSchema.getColumnType(i)) {
case INT:
row[i] = bvIter.nextIntVal();
break;
case FLOAT:
row[i] = bvIter.nextFloatVal();
break;
case LONG:
row[i] = bvIter.nextLongVal();
break;
case DOUBLE:
row[i] = bvIter.nextDoubleVal();
break;
default:
break;
}
}
} else if (blocks[j] instanceof RealtimeMultiValueBlock) {
Dictionary dictionaryReader = blocks[j].getMetadata().getDictionary();
BlockMultiValIterator bvIter =
(BlockMultiValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
int[] dictIds = new int[blocks[j].getMetadata().getMaxNumberOfMultiValues()];
int dictSize;
switch (dataSchema.getColumnType(i)) {
case INT_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
int[] rawIntRow = new int[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawIntRow[dictIdx] =
(Integer) ((IntMutableDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawIntRow;
break;
case FLOAT_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Float[] rawFloatRow = new Float[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawFloatRow[dictIdx] =
(Float) ((FloatMutableDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawFloatRow;
break;
case LONG_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Long[] rawLongRow = new Long[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawLongRow[dictIdx] =
(Long) ((LongMutableDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawLongRow;
break;
case DOUBLE_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Double[] rawDoubleRow = new Double[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawDoubleRow[dictIdx] =
(Double) ((DoubleMutableDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawDoubleRow;
break;
case STRING_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
String[] rawStringRow = new String[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawStringRow[dictIdx] =
(String) (((StringMutableDictionary) dictionaryReader).get(dictIds[dictIdx]));
}
row[i] = rawStringRow;
break;
default:
break;
}
} else if (blocks[j] instanceof UnSortedSingleValueBlock
|| blocks[j] instanceof SortedSingleValueBlock) {
if (blocks[j].getMetadata().hasDictionary()) {
Dictionary dictionaryReader = blocks[j].getMetadata().getDictionary();
BlockSingleValIterator bvIter =
(BlockSingleValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
switch (dataSchema.getColumnType(i)) {
case INT:
int dicId = bvIter.nextIntVal();
row[i] = ((IntDictionary) dictionaryReader).get(dicId);
break;
case FLOAT:
row[i] = ((FloatDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case LONG:
row[i] = ((LongDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case DOUBLE:
row[i] = ((DoubleDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
case STRING:
row[i] = ((StringDictionary) dictionaryReader).get(bvIter.nextIntVal());
break;
default:
break;
}
} else {
BlockSingleValIterator bvIter =
(BlockSingleValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
switch (dataSchema.getColumnType(i)) {
case INT:
row[i] = new Integer(bvIter.nextIntVal());
break;
case FLOAT:
row[i] = new Float(bvIter.nextFloatVal());
break;
case LONG:
row[i] = new Long(bvIter.nextLongVal());
break;
case DOUBLE:
row[i] = new Double(bvIter.nextDoubleVal());
break;
default:
break;
}
}
} else if (blocks[j] instanceof MultiValueBlock) {
Dictionary dictionaryReader = blocks[j].getMetadata().getDictionary();
BlockMultiValIterator bvIter =
(BlockMultiValIterator) blocks[j].getBlockValueSet().iterator();
bvIter.skipTo(docId);
int[] dictIds = new int[blocks[j].getMetadata().getMaxNumberOfMultiValues()];
int dictSize;
switch (dataSchema.getColumnType(i)) {
case INT_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
int[] rawIntRow = new int[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawIntRow[dictIdx] = ((IntDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawIntRow;
break;
case FLOAT_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Float[] rawFloatRow = new Float[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawFloatRow[dictIdx] = ((FloatDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawFloatRow;
break;
case LONG_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Long[] rawLongRow = new Long[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawLongRow[dictIdx] = ((LongDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawLongRow;
break;
case DOUBLE_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
Double[] rawDoubleRow = new Double[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawDoubleRow[dictIdx] = ((DoubleDictionary) dictionaryReader).get(dictIds[dictIdx]);
}
row[i] = rawDoubleRow;
break;
case STRING_ARRAY:
dictSize = bvIter.nextIntVal(dictIds);
String[] rawStringRow = new String[dictSize];
for (int dictIdx = 0; dictIdx < dictSize; ++dictIdx) {
rawStringRow[dictIdx] = (((StringDictionary) dictionaryReader).get(dictIds[dictIdx]));
}
row[i] = rawStringRow;
break;
default:
break;
}
}
j++;
}
return row;
}
public SelectionFetcher(Block[] blocks, DataSchema dataSchema) {
this.length = blocks.length;
selectionColumnIterators = new SelectionColumnIterator[blocks.length];
for (int i = 0; i < dataSchema.size(); ++i) {
if (blocks[i] instanceof RealtimeSingleValueBlock
&& blocks[i].getMetadata().hasDictionary()) {
switch (dataSchema.getColumnType(i)) {
case INT:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Integer, IntMutableDictionary>(
blocks[i]);
break;
case FLOAT:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Float, FloatMutableDictionary>(
blocks[i]);
break;
case LONG:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Long, LongMutableDictionary>(
blocks[i]);
break;
case DOUBLE:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Double, DoubleMutableDictionary>(
blocks[i]);
break;
case STRING:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<String, StringMutableDictionary>(
blocks[i]);
break;
default:
break;
}
} else if ((blocks[i] instanceof UnSortedSingleValueBlock
|| blocks[i] instanceof SortedSingleValueBlock)
&& blocks[i].getMetadata().hasDictionary()) {
switch (dataSchema.getColumnType(i)) {
case INT:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Integer, IntDictionary>(blocks[i]);
break;
case FLOAT:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Float, FloatDictionary>(blocks[i]);
break;
case LONG:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Long, LongDictionary>(blocks[i]);
break;
case DOUBLE:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<Double, DoubleDictionary>(blocks[i]);
break;
case STRING:
selectionColumnIterators[i] =
new SelectionSingleValueColumnWithDictIterator<String, StringDictionary>(blocks[i]);
break;
default:
break;
}
} else if (blocks[i] instanceof RealtimeMultiValueBlock
|| blocks[i] instanceof MultiValueBlock) {
switch (dataSchema.getColumnType(i)) {
case INT_ARRAY:
selectionColumnIterators[i] = new IntArraySelectionColumnIterator(blocks[i]);
break;
case FLOAT_ARRAY:
selectionColumnIterators[i] = new FloatArraySelectionColumnIterator(blocks[i]);
break;
case LONG_ARRAY:
selectionColumnIterators[i] = new LongArraySelectionColumnIterator(blocks[i]);
break;
case DOUBLE_ARRAY:
selectionColumnIterators[i] = new DoubleArraySelectionColumnIterator(blocks[i]);
break;
case STRING_ARRAY:
selectionColumnIterators[i] = new StringArraySelectionColumnIterator(blocks[i]);
break;
default:
break;
}
} else if (!blocks[i].getMetadata().hasDictionary()) {
switch (dataSchema.getColumnType(i)) {
case INT:
selectionColumnIterators[i] = new IntSelectionColumnIterator(blocks[i]);
break;
case FLOAT:
selectionColumnIterators[i] = new FloatSelectionColumnIterator(blocks[i]);
break;
case LONG:
selectionColumnIterators[i] = new LongSelectionColumnIterator(blocks[i]);
break;
case DOUBLE:
selectionColumnIterators[i] = new DoubleSelectionColumnIterator(blocks[i]);
break;
default:
break;
}
} else {
throw new UnsupportedOperationException(
"Failed to get SelectionColumnIterator on Block - " + blocks[i] + " with index - " + i);
}
}
}