private List<Tuple> getAllTableStats(Schema outSchema) {
List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats();
List<Tuple> tuples = new ArrayList<Tuple>(statList.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableStatsProto stat : statList) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("tid".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(stat.getTid()));
} else if ("num_rows".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumRows()));
} else if ("num_bytes".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumBytes()));
}
}
tuples.add(aTuple);
}
return tuples;
}
@Override
public Tuple next() throws IOException {
Tuple aTuple;
Tuple outTuple = new VTuple(outColumnNum);
if (isClosed) {
return null;
}
if (cachedData.size() == 0) {
rescan();
}
if (!scanNode.hasQual()) {
if (currentRow < cachedData.size()) {
aTuple = cachedData.get(currentRow++);
projector.eval(aTuple, outTuple);
outTuple.setOffset(aTuple.getOffset());
return outTuple;
}
return null;
} else {
while (currentRow < cachedData.size()) {
aTuple = cachedData.get(currentRow++);
if (qual.eval(aTuple).isTrue()) {
projector.eval(aTuple, outTuple);
return outTuple;
}
}
return null;
}
}
private List<Tuple> getDatabases(Schema outSchema) {
List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases();
List<Tuple> tuples = new ArrayList<Tuple>(databases.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (DatabaseProto database : databases) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(database.getId()));
} else if ("db_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(database.getName()));
} else if ("space_id".equalsIgnoreCase(column.getSimpleName())) {
if (database.hasSpaceId()) {
aTuple.put(fieldId, DatumFactory.createInt4(database.getSpaceId()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
}
}
tuples.add(aTuple);
}
return tuples;
}
private List<Tuple> getAllTableOptions(Schema outSchema) {
List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions();
List<Tuple> tuples = new ArrayList<Tuple>(optionList.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableOptionProto option : optionList) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("tid".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(option.getTid()));
} else if ("key_".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getKey()));
} else if ("value_".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getValue()));
}
}
tuples.add(aTuple);
}
return tuples;
}
@Override
public Datum eval(Tuple params) {
if (params.isBlankOrNull(0)) {
return NullDatum.get();
}
return DatumFactory.createFloat8(Math.asin(params.getFloat8(0)));
}
@Override
public Datum eval(Tuple params) {
if (params.isBlankOrNull(0)) {
return NullDatum.get();
}
String ipstr = params.getText(0);
return new Int4Datum(IPconvertUtil.ipstr2int(ipstr));
}
@Override
public Datum eval(Tuple params) {
if (params.isBlankOrNull(0) || params.isBlankOrNull(1)) {
return NullDatum.get();
}
String value = params.getText(0);
String pattern = params.getText(1);
TimeMeta tm = DateTimeFormat.parseDateTime(value, pattern);
return new DateDatum(tm);
}
@Override
public Datum eval(Tuple params) {
if (params.isBlankOrNull(0)) {
return NullDatum.get();
}
int value = params.getInt4(0);
if (value <= 0 || value > 65525) {
return NullDatum.get();
} else {
return DatumFactory.createText(String.valueOf((char) value));
}
}
private List<Tuple> getTables(Schema outSchema) {
List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables();
List<Tuple> tuples = new ArrayList<Tuple>(tables.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableDescriptorProto table : tables) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("tid".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(table.getTid()));
} else if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(table.getDbId()));
} else if ("table_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(table.getName()));
} else if ("table_type".equalsIgnoreCase(column.getSimpleName())) {
if (table.hasTableType()) {
aTuple.put(fieldId, DatumFactory.createText(table.getTableType()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("path".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(table.getPath()));
} else if ("store_type".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(table.getStoreType()));
}
}
tuples.add(aTuple);
}
return tuples;
}
public boolean equals(Tuple tuple) {
for (int i = 0; i < keys.length; i++) {
final int field = keyIndex[i];
final boolean n1 = keys[i] == null;
final boolean n2 = tuple.isBlankOrNull(field);
if (n1 && n2) {
continue;
}
if (n1 ^ n2) {
return false;
}
switch (keyTypes[i]) {
case BOOLEAN:
if ((Boolean) keys[i] != tuple.getBool(field)) return false;
continue;
case BIT:
if ((Byte) keys[i] != tuple.getByte(field)) return false;
continue;
case INT1:
case INT2:
if ((Short) keys[i] != tuple.getInt2(field)) return false;
continue;
case INT4:
case DATE:
if ((Integer) keys[i] != tuple.getInt4(field)) return false;
continue;
case INT8:
case TIME:
case TIMESTAMP:
if ((Long) keys[i] != tuple.getInt8(field)) return false;
continue;
case FLOAT4:
if ((Float) keys[i] != tuple.getFloat4(field)) return false;
continue;
case FLOAT8:
if ((Double) keys[i] != tuple.getFloat8(field)) return false;
continue;
case TEXT:
case CHAR:
case BLOB:
if (!Arrays.equals((byte[]) keys[i], tuple.getBytes(field))) return false;
continue;
case DATUM:
if (!keys[i].equals(tuple.asDatum(field))) return false;
continue;
}
}
return true;
}
@Test
public final void testComparatorsFromJoinQual() {
Schema outerSchema = new Schema();
outerSchema.addColumn("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
outerSchema.addColumn("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
Schema innerSchema = new Schema();
innerSchema.addColumn("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
innerSchema.addColumn("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f1 = new FieldEval("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f2 = new FieldEval("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f3 = new FieldEval("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f4 = new FieldEval("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));
EvalNode joinQual = new BinaryEval(EvalType.EQUAL, f1, f2);
TupleComparator[] comparators =
PlannerUtil.getComparatorsFromJoinQual(joinQual, outerSchema, innerSchema);
Tuple t1 = new VTuple(2);
t1.put(0, DatumFactory.createInt4(1));
t1.put(1, DatumFactory.createInt4(2));
Tuple t2 = new VTuple(2);
t2.put(0, DatumFactory.createInt4(2));
t2.put(1, DatumFactory.createInt4(3));
TupleComparator outerComparator = comparators[0];
assertTrue(outerComparator.compare(t1, t2) < 0);
assertTrue(outerComparator.compare(t2, t1) > 0);
TupleComparator innerComparator = comparators[1];
assertTrue(innerComparator.compare(t1, t2) < 0);
assertTrue(innerComparator.compare(t2, t1) > 0);
// tests for composited join key
EvalNode joinQual2 = new BinaryEval(EvalType.EQUAL, f3, f4);
EvalNode compositedJoinQual = new BinaryEval(EvalType.AND, joinQual, joinQual2);
comparators =
PlannerUtil.getComparatorsFromJoinQual(compositedJoinQual, outerSchema, innerSchema);
outerComparator = comparators[0];
assertTrue(outerComparator.compare(t1, t2) < 0);
assertTrue(outerComparator.compare(t2, t1) > 0);
innerComparator = comparators[1];
assertTrue(innerComparator.compare(t1, t2) < 0);
assertTrue(innerComparator.compare(t2, t1) > 0);
}
@Override
public Datum eval(Tuple params) {
Datum datum = params.get(0);
if (datum instanceof NullDatum) return NullDatum.get();
return DatumFactory.createInt4(datum.asByteArray().length * 8);
}
public void set(Tuple tuple) {
for (int i = 0; i < keyTypes.length; i++) {
final int field = keyIndex[i];
if (tuple.isBlankOrNull(field)) {
keys[i] = null;
continue;
}
switch (keyTypes[i]) {
case BOOLEAN:
keys[i] = tuple.getBool(field);
break;
case BIT:
keys[i] = tuple.getByte(field);
break;
case INT1:
case INT2:
keys[i] = tuple.getInt2(field);
break;
case INT4:
case DATE:
keys[i] = tuple.getInt4(field);
break;
case INT8:
case TIME:
case TIMESTAMP:
keys[i] = tuple.getInt8(field);
break;
case FLOAT4:
keys[i] = tuple.getFloat4(field);
break;
case FLOAT8:
keys[i] = tuple.getFloat8(field);
break;
case TEXT:
case CHAR:
case BLOB:
keys[i] = tuple.getBytes(field);
break;
case DATUM:
keys[i] = tuple.asDatum(field);
break;
default:
throw new IllegalArgumentException();
}
}
}
@Override
public Datum eval(Tuple tuple) {
Datum resultDatum;
for (int i = 0; i < tuple.size(); i++) {
params[i] = WritableTypeConverter.convertDatum2Writable(tuple.asDatum(i));
}
try {
Writable result = (Writable) evalMethod.invoke(instance, params);
resultDatum = WritableTypeConverter.convertWritable2Datum(result);
} catch (Exception e) {
throw new TajoInternalError(e);
}
return resultDatum;
}
private List<Tuple> getTablespaces(Schema outSchema) {
List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces();
List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablespaceProto tablespace : tablespaces) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("space_id".equalsIgnoreCase(column.getSimpleName())) {
if (tablespace.hasId()) {
aTuple.put(fieldId, DatumFactory.createInt4(tablespace.getId()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("space_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(tablespace.getSpaceName()));
} else if ("space_handler".equalsIgnoreCase(column.getSimpleName())) {
if (tablespace.hasHandler()) {
aTuple.put(fieldId, DatumFactory.createText(tablespace.getHandler()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("space_uri".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(tablespace.getUri()));
}
}
tuples.add(aTuple);
}
return tuples;
}
@Before
public void setUp() throws Exception {
this.conf = new TajoConf();
util = new TajoTestingCluster();
catalog = util.startCatalogCluster().getCatalog();
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
sm = StorageManagerFactory.getStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerId", Type.INT4);
schema.addColumn("empId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
Appender appender =
StorageManagerFactory.getStorageManager(conf)
.getAppender(employeeMeta, schema, employeePath);
appender.enableStats();
appender.init();
Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < numTuple; i++) {
tuple.put(
new Datum[] {
DatumFactory.createInt4(rnd.nextInt(50)),
DatumFactory.createInt4(rnd.nextInt(100)),
DatumFactory.createText("dept_" + i),
});
appender.addTuple(tuple);
}
appender.flush();
appender.close();
System.out.println(
appender.getStats().getNumRows()
+ " rows ("
+ (appender.getStats().getNumBytes() / 1048576)
+ " MB)");
employee = new TableDesc("employee", schema, employeeMeta, employeePath);
catalog.addTable(employee);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
}
private List<Tuple> getSessionInfo(Schema outSchema) {
List<Tuple> outputs = Lists.newArrayList();
Tuple eachVariable;
try {
for (Map.Entry<String, String> var :
masterContext.getSessionManager().getAllVariables(sessionId).entrySet()) {
eachVariable = new VTuple(outSchema.size());
eachVariable.put(0, DatumFactory.createText(var.getKey()));
eachVariable.put(1, DatumFactory.createText(var.getValue()));
outputs.add(eachVariable);
}
} catch (InvalidSessionException e) {
LOG.error(e);
}
return outputs;
}
/**
* It computes the value cardinality of a tuple range.
*
* @return
*/
public static BigDecimal computeCardinalityForAllColumns(
Schema schema, TupleRange range, boolean inclusive) {
Tuple start = range.getStart();
Tuple end = range.getEnd();
Column col;
BigDecimal cardinality = new BigDecimal(1);
BigDecimal columnCard;
for (int i = 0; i < schema.getColumnNum(); i++) {
col = schema.getColumn(i);
columnCard = computeCardinality(col.getDataType(), start.get(i), end.get(i), inclusive);
if (new BigDecimal(0).compareTo(columnCard) < 0) {
cardinality = cardinality.multiply(columnCard);
}
}
return cardinality;
}
/** Called after all fields have been processed. */
@Override
public void end() {
for (int i = 0; i < projectionMap.length; ++i) {
final int projectionIndex = projectionMap[i];
Column column = tajoReadSchema.getColumn(projectionIndex);
if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE
|| currentTuple.isBlankOrNull(i)) {
set(projectionIndex, NullDatum.get());
}
}
}
private void writeValue(Column column, Tuple tuple, int index) {
switch (column.getDataType().getType()) {
case BOOLEAN:
recordConsumer.addBoolean(tuple.getBool(index));
break;
case BIT:
case INT2:
case INT4:
recordConsumer.addInteger(tuple.getInt4(index));
break;
case INT8:
recordConsumer.addLong(tuple.getInt8(index));
break;
case FLOAT4:
recordConsumer.addFloat(tuple.getFloat4(index));
break;
case FLOAT8:
recordConsumer.addDouble(tuple.getFloat8(index));
break;
case CHAR:
if (tuple.size(index) > column.getDataType().getLength()) {
throw new ValueTooLongForTypeCharactersException(column.getDataType().getLength());
}
recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
break;
case TEXT:
recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
break;
case DATE:
// Parquet DATE type is based on Unix Epoch(Jan 1, 1970).
recordConsumer.addInteger(tuple.getInt4(index) - DateTimeConstants.UNIX_EPOCH_JDATE);
break;
case PROTOBUF:
case BLOB:
recordConsumer.addBinary(Binary.fromByteArray(tuple.getBytes(index)));
break;
default:
break;
}
}
private List<Tuple> getAllPartitions(Schema outSchema) {
List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions();
List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablePartitionProto partition : partitionList) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("partition_id".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(partition.getPartitionId()));
} else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(partition.getTid()));
} else if ("partition_name".equalsIgnoreCase(column.getSimpleName())) {
if (partition.hasPartitionName()) {
aTuple.put(fieldId, DatumFactory.createText(partition.getPartitionName()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("path".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(partition.getPath()));
}
}
tuples.add(aTuple);
}
return tuples;
}
@Override
public Datum eval(Tuple params) {
if (params.isBlankOrNull(0) || params.isBlankOrNull(1)) {
return NullDatum.get();
}
String fillText;
if (hasFillCharacters) {
fillText = params.getText(2);
} else {
fillText = " ";
}
String input = params.getText(0);
int expected = params.getInt4(1);
int templen = expected - params.size(0);
if (templen <= 0) {
return DatumFactory.createText(input.substring(0, expected));
} else {
return DatumFactory.createText(StringUtils.leftPad(input, expected, fillText));
}
}
private List<Tuple> getIndexes(Schema outSchema) {
List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (IndexProto index : indexList) {
aTuple = new VTuple(outSchema.size());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId()));
} else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(index.getTId()));
} else if ("index_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getIndexName()));
} else if ("column_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getColumnName()));
} else if ("data_type".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getDataType()));
} else if ("index_type".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getIndexType()));
} else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique()));
} else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered()));
} else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending()));
}
}
tuples.add(aTuple);
}
return tuples;
}
protected final void set(int index, Tuple tuple, int field) {
if (tuple.isBlankOrNull(field)) {
nulls.set(index);
return;
}
nulls.clear(index);
switch (type) {
case 0:
booleans[index] = tuple.getBool(field);
break;
case 1:
bits[index] = tuple.getByte(field);
break;
case 2:
shorts[index] = tuple.getInt2(field);
break;
case 3:
ints[index] = tuple.getInt4(field);
break;
case 4:
longs[index] = tuple.getInt8(field);
break;
case 5:
floats[index] = tuple.getFloat4(field);
break;
case 6:
doubles[index] = tuple.getFloat8(field);
break;
case 7:
bytes[index] = tuple.getBytes(field);
break;
case 8:
ints[index] = tuple.getInt4(field);
break;
default:
throw new IllegalArgumentException();
}
}
private List<Tuple> getColumns(Schema outSchema) {
List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns();
List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size());
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
int columnId = 1, prevtid = -1, tid = 0;
for (ColumnProto column : columnsList) {
aTuple = new VTuple(outSchema.size());
tid = column.getTid();
if (prevtid != tid) {
columnId = 1;
prevtid = tid;
}
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column colObj = columns.get(fieldId);
if ("tid".equalsIgnoreCase(colObj.getSimpleName())) {
if (column.hasTid()) {
aTuple.put(fieldId, DatumFactory.createInt4(tid));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("column_name".equalsIgnoreCase(colObj.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(column.getName()));
} else if ("ordinal_position".equalsIgnoreCase(colObj.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createInt4(columnId));
} else if ("data_type".equalsIgnoreCase(colObj.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(column.getDataType().getType().toString()));
} else if ("type_length".equalsIgnoreCase(colObj.getSimpleName())) {
DataType dataType = column.getDataType();
if (dataType.hasLength()) {
aTuple.put(fieldId, DatumFactory.createInt4(dataType.getLength()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
}
}
columnId++;
tuples.add(aTuple);
}
return tuples;
}
private void writeRecordFields(GroupType schema, Schema tajoSchema, Tuple tuple) {
List<Type> fields = schema.getFields();
// Parquet ignores Tajo NULL_TYPE columns, so the index may differ.
int index = 0;
for (int tajoIndex = 0; tajoIndex < tajoSchema.size(); ++tajoIndex) {
Column column = tajoSchema.getColumn(tajoIndex);
if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) {
continue;
}
Type fieldType = fields.get(index);
if (!tuple.isBlankOrNull(tajoIndex)) {
recordConsumer.startField(fieldType.getName(), index);
writeValue(column, tuple, tajoIndex);
recordConsumer.endField(fieldType.getName(), index);
} else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) {
throw new RuntimeException("Null-value for required field: " + column.getSimpleName());
}
++index;
}
}
@Override
public void merge(FunctionContext ctx, Tuple part) {
((CountRowContext) ctx).count += part.get(0).asInt8();
}
public Tuple next() throws IOException {
Tuple previous;
for (; ; ) {
if (!outerIterator.hasNext() && !innerIterator.hasNext()) {
if (end) {
return null;
}
if (outerTuple == null) {
outerTuple = leftChild.next();
}
if (innerTuple == null) {
innerTuple = rightChild.next();
}
outerTupleSlots.clear();
innerTupleSlots.clear();
int cmp;
while ((cmp = joincomparator.compare(outerTuple, innerTuple)) != 0) {
if (cmp > 0) {
innerTuple = rightChild.next();
} else if (cmp < 0) {
outerTuple = leftChild.next();
}
if (innerTuple == null || outerTuple == null) {
return null;
}
}
try {
previous = outerTuple.clone();
do {
outerTupleSlots.add(outerTuple.clone());
outerTuple = leftChild.next();
if (outerTuple == null) {
end = true;
break;
}
} while (tupleComparator[0].compare(previous, outerTuple) == 0);
outerIterator = outerTupleSlots.iterator();
outerNext = outerIterator.next();
previous = innerTuple.clone();
do {
innerTupleSlots.add(innerTuple.clone());
innerTuple = rightChild.next();
if (innerTuple == null) {
end = true;
break;
}
} while (tupleComparator[1].compare(previous, innerTuple) == 0);
innerIterator = innerTupleSlots.iterator();
} catch (CloneNotSupportedException e) {
}
}
if (!innerIterator.hasNext()) {
outerNext = outerIterator.next();
innerIterator = innerTupleSlots.iterator();
}
frameTuple.set(outerNext, innerIterator.next());
joinQual.eval(qualCtx, inSchema, frameTuple);
if (joinQual.terminate(qualCtx).asBool()) {
projector.eval(evalContexts, frameTuple);
projector.terminate(evalContexts, outTuple);
return outTuple;
}
}
}
private void set(int index, Object value) {
currentTuple.put(index, (Datum) value);
}
@Before
public void setup() throws Exception {
this.randomValues = new HashMap<Integer, Integer>();
this.conf = new TajoConf();
util = new TajoTestingCluster();
util.startCatalogCluster();
catalog = util.getMiniCatalogCluster().getCatalog();
Path workDir = CommonTestingUtil.getTestDir();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
sm = StorageManagerFactory.getStorageManager(conf, workDir);
idxPath = new Path(workDir, "test.idx");
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
schema.addColumn("empid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
this.idxSchema = new Schema();
idxSchema.addColumn("managerid", Type.INT4);
SortSpec[] sortKeys = new SortSpec[1];
sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false);
this.comp = new TupleComparator(idxSchema, sortKeys);
this.writer =
new BSTIndex(conf)
.getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp);
writer.setLoadNum(100);
writer.open();
long offset;
meta = CatalogUtil.newTableMeta(StoreType.CSV);
tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
fs = tablePath.getFileSystem(conf);
fs.mkdirs(tablePath.getParent());
FileAppender appender =
(FileAppender)
StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple = new VTuple(schema.size());
for (int i = 0; i < 10000; i++) {
Tuple key = new VTuple(this.idxSchema.size());
int rndKey = rnd.nextInt(250);
if (this.randomValues.containsKey(rndKey)) {
int t = this.randomValues.remove(rndKey) + 1;
this.randomValues.put(rndKey, t);
} else {
this.randomValues.put(rndKey, 1);
}
key.put(new Datum[] {DatumFactory.createInt4(rndKey)});
tuple.put(
new Datum[] {
DatumFactory.createInt4(rndKey),
DatumFactory.createInt4(rnd.nextInt(10)),
DatumFactory.createText("dept_" + rnd.nextInt(10))
});
offset = appender.getOffset();
appender.addTuple(tuple);
writer.write(key, offset);
}
appender.flush();
appender.close();
writer.close();
TableDesc desc =
new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"),
schema,
meta,
sm.getTablePath("employee"));
catalog.createTable(desc);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
optimizer = new LogicalOptimizer(conf);
}
