private List<Tuple> getAllPartitions(Schema outSchema) { List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions(); List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TablePartitionProto partition : partitionList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("partition_id".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(partition.getPartitionId())); } else if ("tid".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(partition.getTid())); } else if ("partition_name".equalsIgnoreCase(column.getSimpleName())) { if (partition.hasPartitionName()) { aTuple.put(fieldId, DatumFactory.createText(partition.getPartitionName())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("path".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(partition.getPath())); } } tuples.add(aTuple); } return tuples; }
/** * Creates a new TajoRecordConverter. * * @param parquetSchema The Parquet schema of the projection. * @param tajoReadSchema The Tajo schema of the table. * @param projectionMap An array mapping the projection column to the column index in the table. */ public TajoRecordConverter(GroupType parquetSchema, Schema tajoReadSchema, int[] projectionMap) { this.parquetSchema = parquetSchema; this.tajoReadSchema = tajoReadSchema; this.projectionMap = projectionMap; this.tupleSize = tajoReadSchema.size(); // The projectionMap.length does not match parquetSchema.getFieldCount() // when the projection contains NULL_TYPE columns. We will skip over the // NULL_TYPE columns when we construct the converters and populate the // NULL_TYPE columns with NullDatums in start(). int index = 0; this.converters = new Converter[parquetSchema.getFieldCount()]; for (int i = 0; i < projectionMap.length; ++i) { final int projectionIndex = projectionMap[i]; Column column = tajoReadSchema.getColumn(projectionIndex); if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) { continue; } Type type = parquetSchema.getType(index); final int writeIndex = i; converters[index] = newConverter( column, type, new ParentValueContainer() { @Override void add(Object value) { TajoRecordConverter.this.set(writeIndex, value); } }); ++index; } }
private List<Tuple> getAllTableOptions(Schema outSchema) { List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions(); List<Tuple> tuples = new ArrayList<Tuple>(optionList.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableOptionProto option : optionList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("tid".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(option.getTid())); } else if ("key_".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getKey())); } else if ("value_".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getValue())); } } tuples.add(aTuple); } return tuples; }
private List<Tuple> getAllTableStats(Schema outSchema) { List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats(); List<Tuple> tuples = new ArrayList<Tuple>(statList.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableStatsProto stat : statList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("tid".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(stat.getTid())); } else if ("num_rows".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumRows())); } else if ("num_bytes".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumBytes())); } } tuples.add(aTuple); } return tuples; }
private List<Tuple> getTables(Schema outSchema) { List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables(); List<Tuple> tuples = new ArrayList<Tuple>(tables.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TableDescriptorProto table : tables) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("tid".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(table.getTid())); } else if ("db_id".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(table.getDbId())); } else if ("table_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getName())); } else if ("table_type".equalsIgnoreCase(column.getSimpleName())) { if (table.hasTableType()) { aTuple.put(fieldId, DatumFactory.createText(table.getTableType())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("path".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getPath())); } else if ("store_type".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getStoreType())); } } tuples.add(aTuple); } return tuples; }
private List<Tuple> getDatabases(Schema outSchema) { List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases(); List<Tuple> tuples = new ArrayList<Tuple>(databases.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (DatabaseProto database : databases) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("db_id".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(database.getId())); } else if ("db_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(database.getName())); } else if ("space_id".equalsIgnoreCase(column.getSimpleName())) { if (database.hasSpaceId()) { aTuple.put(fieldId, DatumFactory.createInt4(database.getSpaceId())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } } tuples.add(aTuple); } return tuples; }
private List<Tuple> getTablespaces(Schema outSchema) { List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces(); List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (TablespaceProto tablespace : tablespaces) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("space_id".equalsIgnoreCase(column.getSimpleName())) { if (tablespace.hasId()) { aTuple.put(fieldId, DatumFactory.createInt4(tablespace.getId())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("space_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(tablespace.getSpaceName())); } else if ("space_handler".equalsIgnoreCase(column.getSimpleName())) { if (tablespace.hasHandler()) { aTuple.put(fieldId, DatumFactory.createText(tablespace.getHandler())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("space_uri".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(tablespace.getUri())); } } tuples.add(aTuple); } return tuples; }
private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) { List<Column> columns = outSchema.getRootColumns(); Tuple aTuple = new VTuple(outSchema.size()); WorkerResource aResource = aWorker.getResource(); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("host".equalsIgnoreCase(column.getSimpleName())) { if (aWorker.getConnectionInfo() != null && aWorker.getConnectionInfo().getHost() != null) { aTuple.put(fieldId, DatumFactory.createText(aWorker.getConnectionInfo().getHost())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("port".equalsIgnoreCase(column.getSimpleName())) { if (aWorker.getConnectionInfo() != null) { aTuple.put( fieldId, DatumFactory.createInt4(aWorker.getConnectionInfo().getPeerRpcPort())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("type".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText("Worker")); } else if ("status".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(aWorker.getState().toString())); } else if ("RUNNING".equalsIgnoreCase(aWorker.getState().toString())) { if ("total_cpu".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(aResource.getCpuCoreSlots())); } else if ("used_mem".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(aResource.getUsedMemoryMB() * 1048576l)); } else if ("total_mem".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(aResource.getMemoryMB() * 1048576l)); } else if ("free_heap".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(aResource.getFreeHeap())); } else if ("max_heap".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt8(aResource.getMaxHeap())); } else if ("used_diskslots".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createFloat4(aResource.getUsedDiskSlots())); } else if ("total_diskslots".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createFloat4(aResource.getDiskSlots())); } else if ("running_tasks".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(aResource.getNumRunningTasks())); } else if ("last_heartbeat_ts".equalsIgnoreCase(column.getSimpleName())) { if (aWorker.getLastHeartbeatTime() > 0) { aTuple.put( fieldId, DatumFactory.createTimestmpDatumWithJavaMillis(aWorker.getLastHeartbeatTime())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } return aTuple; }
private boolean isClonedSchema(Schema origSchema, Schema newSchema) { // Check schema of tables boolean schemaEqual = (origSchema.size() == newSchema.size()); if (schemaEqual == false) { fail("Number of columns in schema not equal"); return false; } for (int col = 0; col < origSchema.size(); col++) { Column colA = origSchema.getColumn(col); Column colB = newSchema.getColumn(col); if (colA.getSimpleName().equals(colB.getSimpleName()) == false) { fail("Column names at index " + col + " do not match"); return false; } if (colA.getDataType().equals(colB.getDataType()) == false) { fail("Column datatypes at index " + col + " do not match"); return false; } } return true; }
private List<Tuple> getColumns(Schema outSchema) { List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns(); List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; int columnId = 1, prevtid = -1, tid = 0; for (ColumnProto column : columnsList) { aTuple = new VTuple(outSchema.size()); tid = column.getTid(); if (prevtid != tid) { columnId = 1; prevtid = tid; } for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column colObj = columns.get(fieldId); if ("tid".equalsIgnoreCase(colObj.getSimpleName())) { if (column.hasTid()) { aTuple.put(fieldId, DatumFactory.createInt4(tid)); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } else if ("column_name".equalsIgnoreCase(colObj.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(column.getName())); } else if ("ordinal_position".equalsIgnoreCase(colObj.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(columnId)); } else if ("data_type".equalsIgnoreCase(colObj.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(column.getDataType().getType().toString())); } else if ("type_length".equalsIgnoreCase(colObj.getSimpleName())) { DataType dataType = column.getDataType(); if (dataType.hasLength()) { aTuple.put(fieldId, DatumFactory.createInt4(dataType.getLength())); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } } } columnId++; tuples.add(aTuple); } return tuples; }
private List<Tuple> getSessionInfo(Schema outSchema) { List<Tuple> outputs = Lists.newArrayList(); Tuple eachVariable; try { for (Map.Entry<String, String> var : masterContext.getSessionManager().getAllVariables(sessionId).entrySet()) { eachVariable = new VTuple(outSchema.size()); eachVariable.put(0, DatumFactory.createText(var.getKey())); eachVariable.put(1, DatumFactory.createText(var.getValue())); outputs.add(eachVariable); } } catch (InvalidSessionException e) { LOG.error(e); } return outputs; }
private void writeRecordFields(GroupType schema, Schema tajoSchema, Tuple tuple) { List<Type> fields = schema.getFields(); // Parquet ignores Tajo NULL_TYPE columns, so the index may differ. int index = 0; for (int tajoIndex = 0; tajoIndex < tajoSchema.size(); ++tajoIndex) { Column column = tajoSchema.getColumn(tajoIndex); if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) { continue; } Type fieldType = fields.get(index); if (!tuple.isBlankOrNull(tajoIndex)) { recordConsumer.startField(fieldType.getName(), index); writeValue(column, tuple, tajoIndex); recordConsumer.endField(fieldType.getName(), index); } else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) { throw new RuntimeException("Null-value for required field: " + column.getSimpleName()); } ++index; } }
private List<Tuple> getIndexes(Schema outSchema) { List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes(); List<Tuple> tuples = new ArrayList<Tuple>(indexList.size()); List<Column> columns = outSchema.getRootColumns(); Tuple aTuple; for (IndexProto index : indexList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); if ("db_id".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId())); } else if ("tid".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(index.getTId())); } else if ("index_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(index.getIndexName())); } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(index.getColumnName())); } else if ("data_type".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(index.getDataType())); } else if ("index_type".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(index.getIndexType())); } else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique())); } else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered())); } else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending())); } } tuples.add(aTuple); } return tuples; }
@Before public void setUp() throws Exception { util = new TajoTestingCluster(); util.initTestDir(); util.startCatalogCluster(); catalog = util.getCatalogService(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); // ----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 Schema dep3Schema = new Schema(); dep3Schema.addColumn("dep_id", Type.INT4); dep3Schema.addColumn("dep_name", Type.TEXT); dep3Schema.addColumn("loc_id", Type.INT4); TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT"); Path dep3Path = new Path(testDir, "dep3.csv"); Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); VTuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { tuple.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender1.addTuple(tuple); } appender1.flush(); appender1.close(); dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path); catalog.createTable(dep3); // ----------------- dep4 ------------------------------ // dep_id | dep_name | loc_id // -------------------------------- // 0 | dep_0 | 1000 // 1 | dep_1 | 1001 // 2 | dep_2 | 1002 // 3 | dep_3 | 1003 // 4 | dep_4 | 1004 // 5 | dep_5 | 1005 // 6 | dep_6 | 1006 // 7 | dep_7 | 1007 // 8 | dep_8 | 1008 // 9 | dep_9 | 1009 // 10 | dep_10 | 1010 Schema dep4Schema = new Schema(); dep4Schema.addColumn("dep_id", Type.INT4); dep4Schema.addColumn("dep_name", Type.TEXT); dep4Schema.addColumn("loc_id", Type.INT4); TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT"); Path dep4Path = new Path(testDir, "dep4.csv"); Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); VTuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { tuple4.put( new Datum[] { DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i), DatumFactory.createInt4(1000 + i) }); appender4.addTuple(tuple4); } appender4.flush(); appender4.close(); dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path); catalog.createTable(dep4); // ----------------- job3 ------------------------------ // job_id | job_title // ---------------------- // 101 | job_101 // 102 | job_102 // 103 | job_103 Schema job3Schema = new Schema(); job3Schema.addColumn("job_id", Type.INT4); job3Schema.addColumn("job_title", Type.TEXT); TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT"); Path job3Path = new Path(testDir, "job3.csv"); Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(job3Meta, job3Schema, job3Path); appender2.init(); VTuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { int x = 100 + i; tuple2.put( new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)}); appender2.addTuple(tuple2); } appender2.flush(); appender2.close(); job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path); catalog.createTable(job3); // ---------------------emp3 -------------------- // emp_id | first_name | last_name | dep_id | salary | job_id // ------------------------------------------------------------ // 11 | fn_11 | ln_11 | 1 | 123 | 101 // 13 | fn_13 | ln_13 | 3 | 369 | 103 // 15 | fn_15 | ln_15 | 5 | 615 | null // 17 | fn_17 | ln_17 | 7 | 861 | null // 19 | fn_19 | ln_19 | 9 | 1107 | null // 21 | fn_21 | ln_21 | 1 | 123 | 101 // 23 | fn_23 | ln_23 | 3 | 369 | 103 Schema emp3Schema = new Schema(); emp3Schema.addColumn("emp_id", Type.INT4); emp3Schema.addColumn("first_name", Type.TEXT); emp3Schema.addColumn("last_name", Type.TEXT); emp3Schema.addColumn("dep_id", Type.INT4); emp3Schema.addColumn("salary", Type.FLOAT4); emp3Schema.addColumn("job_id", Type.INT4); TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT"); Path emp3Path = new Path(testDir, "emp3.csv"); Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); VTuple tuple3 = new VTuple(emp3Schema.size()); for (int i = 1; i < 4; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); int y = 20 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(20 + i), DatumFactory.createText("firstname_" + y), DatumFactory.createText("lastname_" + y), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createInt4(100 + i) }); appender3.addTuple(tuple3); } for (int i = 5; i < 10; i += 2) { int x = 10 + i; tuple3.put( new Datum[] { DatumFactory.createInt4(10 + i), DatumFactory.createText("firstname_" + x), DatumFactory.createText("lastname_" + x), DatumFactory.createInt4(i), DatumFactory.createFloat4(123 * i), DatumFactory.createNullDatum() }); appender3.addTuple(tuple3); } appender3.flush(); appender3.close(); emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path); catalog.createTable(emp3); // ---------------------phone3 -------------------- // emp_id | phone_number // ----------------------------------------------- // this table is empty, no rows Schema phone3Schema = new Schema(); phone3Schema.addColumn("emp_id", Type.INT4); phone3Schema.addColumn("phone_number", Type.TEXT); TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT"); Path phone3Path = new Path(testDir, "phone3.csv"); Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); appender5.close(); phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); defaultContext = LocalTajoTestingUtility.createDummyContext(conf); }
@Before public void setup() throws Exception { this.randomValues = new HashMap<Integer, Integer>(); this.conf = new TajoConf(); util = new TajoTestingCluster(); util.startCatalogCluster(); catalog = util.getMiniCatalogCluster().getCatalog(); Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); sm = StorageManagerFactory.getStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); schema.addColumn("empid", Type.INT4); schema.addColumn("deptname", Type.TEXT); this.idxSchema = new Schema(); idxSchema.addColumn("managerid", Type.INT4); SortSpec[] sortKeys = new SortSpec[1]; sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false); this.comp = new TupleComparator(idxSchema, sortKeys); this.writer = new BSTIndex(conf) .getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp); writer.setLoadNum(100); writer.open(); long offset; meta = CatalogUtil.newTableMeta(StoreType.CSV); tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 10000; i++) { Tuple key = new VTuple(this.idxSchema.size()); int rndKey = rnd.nextInt(250); if (this.randomValues.containsKey(rndKey)) { int t = this.randomValues.remove(rndKey) + 1; this.randomValues.put(rndKey, t); } else { this.randomValues.put(rndKey, 1); } key.put(new Datum[] {DatumFactory.createInt4(rndKey)}); tuple.put( new Datum[] { DatumFactory.createInt4(rndKey), DatumFactory.createInt4(rnd.nextInt(10)), DatumFactory.createText("dept_" + rnd.nextInt(10)) }); offset = appender.getOffset(); appender.addTuple(tuple); writer.write(key, offset); } appender.flush(); appender.close(); writer.close(); TableDesc desc = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, sm.getTablePath("employee")); catalog.createTable(desc); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); }
@Test public void testColumnKeyValueMapping() throws Exception { KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.set(HBaseStorageConstants.META_TABLE_KEY, "test"); keyValueSet.set(HBaseStorageConstants.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:"); Schema schema = new Schema(); schema.addColumn("c1", Type.TEXT); schema.addColumn("c2", Type.TEXT); schema.addColumn("c3", Type.TEXT); schema.addColumn("c4", Type.TEXT); TableMeta tableMeta = new TableMeta("HBASE", keyValueSet); ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getPropertySet()); List<String> cfNames = columnMapping.getColumnFamilyNames(); assertEquals(2, cfNames.size()); assertEquals("col2", cfNames.get(0)); assertEquals("col3", cfNames.get(1)); for (int i = 0; i < columnMapping.getIsBinaryColumns().length; i++) { if (i == 2) { assertTrue(columnMapping.getIsBinaryColumns()[i]); } else { assertFalse(columnMapping.getIsBinaryColumns()[i]); } } for (int i = 0; i < columnMapping.getIsRowKeyMappings().length; i++) { if (i == 0) { assertTrue(columnMapping.getIsRowKeyMappings()[i]); } else { assertFalse(columnMapping.getIsRowKeyMappings()[i]); } } String[] expectedColumnNames = {null, null, null, null}; for (int i = 0; i < schema.size(); i++) { String columnName = columnMapping.getMappingColumns()[i][1] == null ? null : new String(columnMapping.getMappingColumns()[i][1]); assertEquals(expectedColumnNames[i], columnName); } for (int i = 0; i < schema.size(); i++) { if (i == 1) { assertTrue(columnMapping.getIsColumnKeys()[i]); } else { assertFalse(columnMapping.getIsColumnKeys()[i]); } } for (int i = 0; i < schema.size(); i++) { if (i == 2) { assertTrue(columnMapping.getIsColumnValues()[i]); } else { assertFalse(columnMapping.getIsColumnValues()[i]); } } }