@Override
public ReturnState createTable(RpcController controller, TableDescProto request) {
String[] splitted = CatalogUtil.splitFQTableName(request.getTableName());
String dbName = splitted[0];
String tbName = splitted[1];
if (linkedMetadataManager.existsDatabase(dbName)) {
return errInsufficientPrivilege("drop a table in database '" + dbName + "'");
}
if (metaDictionary.isSystemDatabase(dbName)) {
return errInsufficientPrivilege("create a table in database '" + dbName + "'");
}
wlock.lock();
try {
store.createTable(request);
LOG.info(
String.format(
"relation \"%s\" is added to the catalog (%s)",
CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddressStr));
return OK;
} catch (Throwable t) {
printStackTraceIfError(LOG, t);
return returnError(t);
} finally {
wlock.unlock();
}
}
@Test
public final void testRegisterAndFindFunc() throws Exception {
assertFalse(catalog.containFunction("test10", FunctionType.GENERAL));
FunctionDesc meta =
new FunctionDesc(
"test10",
TestFunc2.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB));
catalog.createFunction(meta);
assertTrue(
catalog.containFunction(
"test10", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB)));
FunctionDesc retrived =
catalog.getFunction("test10", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB));
assertEquals(retrived.getFunctionName(), "test10");
assertEquals(retrived.getLegacyFuncClass(), TestFunc2.class);
assertEquals(retrived.getFuncType(), FunctionType.GENERAL);
assertFalse(
catalog.containFunction(
"test10", CatalogUtil.newSimpleDataTypeArray(Type.BLOB, Type.INT4)));
}
@Test
public final void testFindAnyTypeParamFunc() throws Exception {
assertFalse(catalog.containFunction("testany", FunctionType.GENERAL));
FunctionDesc meta =
new FunctionDesc(
"testany",
TestAnyParamFunc.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.ANY));
catalog.createFunction(meta);
FunctionDesc retrieved =
catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.INT1));
assertEquals(retrieved.getFunctionName(), "testany");
assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY));
retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.INT8));
assertEquals(retrieved.getFunctionName(), "testany");
assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY));
retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4));
assertEquals(retrieved.getFunctionName(), "testany");
assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY));
retrieved = catalog.getFunction("testany", CatalogUtil.newSimpleDataTypeArray(Type.TEXT));
assertEquals(retrieved.getFunctionName(), "testany");
assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.ANY));
}
@Test
public final void testDropFunction() throws Exception {
assertFalse(catalog.containFunction("test3", CatalogUtil.newSimpleDataTypeArray(Type.INT4)));
FunctionDesc meta =
new FunctionDesc(
"test3",
TestFunc1.class,
FunctionType.UDF,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.INT4));
catalog.createFunction(meta);
assertTrue(catalog.containFunction("test3", CatalogUtil.newSimpleDataTypeArray(Type.INT4)));
catalog.dropFunction("test3");
assertFalse(catalog.containFunction("test3", CatalogUtil.newSimpleDataTypeArray(Type.INT4)));
assertFalse(
catalog.containFunction("test3", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB)));
FunctionDesc overload =
new FunctionDesc(
"test3",
TestFunc2.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB));
catalog.createFunction(overload);
assertTrue(
catalog.containFunction("test3", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.BLOB)));
}
@Test
public final void testSuchFunctionException() throws Exception {
try {
assertFalse(
catalog.containFunction("test123", CatalogUtil.newSimpleDataTypeArray(Type.INT4)));
catalog.getFunction("test123", CatalogUtil.newSimpleDataTypeArray(Type.INT4));
fail();
} catch (UndefinedFunctionException nsfe) {
// succeed test
} catch (Throwable e) {
fail(e.getMessage());
}
}
@Test(expected = UndefinedFunctionException.class)
public final void testFindIntInvalidFunc() throws Exception {
assertFalse(catalog.containFunction("testintinvalid", FunctionType.GENERAL));
FunctionDesc meta =
new FunctionDesc(
"testintinvalid",
TestIntFunc.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT4));
catalog.createFunction(meta);
// UPGRADE TO INT8 WILL FAIL ==> LOOK AT SECOND PARAM BELOW
catalog.getFunction("testintinvalid", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8));
}
@Test(expected = UndefinedFunctionException.class)
public final void testFindFloatInvalidFunc() throws Exception {
assertFalse(catalog.containFunction("testfloatinvalid", FunctionType.GENERAL));
FunctionDesc meta =
new FunctionDesc(
"testfloatinvalid",
TestFloatFunc.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.FLOAT8, Type.INT4));
catalog.createFunction(meta);
// UPGRADE TO DECIMAL WILL FAIL ==> LOOK AT FIRST PARAM BELOW
catalog.getFunction(
"testfloatinvalid", CatalogUtil.newSimpleDataTypeArray(Type.NUMERIC, Type.INT4));
}
@Override
public ReturnState dropTable(RpcController controller, TableIdentifierProto request)
throws ServiceException {
String dbName = request.getDatabaseName();
String tbName = request.getTableName();
if (linkedMetadataManager.existsDatabase(dbName)) {
return errInsufficientPrivilege("drop a table in database '" + dbName + "'");
}
if (metaDictionary.isSystemDatabase(dbName)) {
return errInsufficientPrivilege("drop a table in database '" + dbName + "'");
}
wlock.lock();
try {
store.dropTable(dbName, tbName);
LOG.info(
String.format(
"relation \"%s\" is deleted from the catalog (%s)",
CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddressStr));
return OK;
} catch (Throwable t) {
printStackTraceIfError(LOG, t);
return returnError(t);
} finally {
wlock.unlock();
}
}
@Override
public ReturnState alterTable(RpcController controller, AlterTableDescProto proto) {
String[] split = CatalogUtil.splitTableName(proto.getTableName());
if (linkedMetadataManager.existsDatabase(split[0])) {
return errInsufficientPrivilege("alter a table in database '" + split[0] + "'");
}
if (metaDictionary.isSystemDatabase(split[0])) {
return errInsufficientPrivilege("alter a table in database '" + split[0] + "'");
}
wlock.lock();
try {
store.alterTable(proto);
return OK;
} catch (Throwable t) {
printStackTraceIfError(LOG, t);
return returnError(t);
} finally {
wlock.unlock();
}
}
private FunctionDescProto findFunction(
String signature,
FunctionType type,
List<TajoDataTypes.DataType> params,
boolean strictTypeCheck) {
List<FunctionDescProto> candidates = Lists.newArrayList();
if (functions.containsKey(signature)) {
if (strictTypeCheck) {
for (FunctionDescProto func : functions.get(signature)) {
if (func.getSignature().getType() == type
&& func.getSignature().getParameterTypesList().equals(params)) {
candidates.add(func);
}
}
} else {
for (FunctionDescProto func : functions.get(signature)) {
if (func.getSignature().getParameterTypesList() != null
&& CatalogUtil.isMatchedFunction(
func.getSignature().getParameterTypesList(), params)) {
candidates.add(func);
}
}
}
}
// if there are more than one function candidates, we choose the nearest matched function.
if (candidates.size() > 0) {
return findNearestMatchedFunction(candidates);
} else {
return null;
}
}
@Test
public final void testFindIntFunc() throws Exception {
assertFalse(catalog.containFunction("testint", FunctionType.GENERAL));
FunctionDesc meta =
new FunctionDesc(
"testint",
TestIntFunc.class,
FunctionType.GENERAL,
CatalogUtil.newSimpleDataType(Type.INT4),
CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT4));
catalog.createFunction(meta);
// UPGRADE TO INT4 SUCCESS==> LOOK AT SECOND PARAM BELOW
FunctionDesc retrieved =
catalog.getFunction("testint", CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT2));
assertEquals(retrieved.getFunctionName(), "testint");
assertEquals(retrieved.getParamTypes()[0], CatalogUtil.newSimpleDataType(Type.INT4));
assertEquals(retrieved.getParamTypes()[1], CatalogUtil.newSimpleDataType(Type.INT4));
}
public static TableDesc prepareTable() throws IOException {
relationSchema =
SchemaBuilder.builder()
.add(DEFAULT_DATABASE_NAME + ".indexed.id", Type.INT4)
.add(DEFAULT_DATABASE_NAME + ".indexed.name", Type.TEXT)
.add(DEFAULT_DATABASE_NAME + ".indexed.age", Type.INT4)
.add(DEFAULT_DATABASE_NAME + ".indexed.score", Type.FLOAT8)
.build();
String tableName = "indexed";
TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, server.getConf());
return new TableDesc(
IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName),
relationSchema,
meta,
new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
}
@Test
public final void testAddAndDeleteTablePartitionByRange() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String tableName =
IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).build();
PartitionMethodDesc partitionDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME, tableName, CatalogProtos.PartitionType.RANGE, "id", partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.RANGE);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
private FunctionDescProto findFunction(String signature, List<TajoDataTypes.DataType> params) {
List<FunctionDescProto> candidates = Lists.newArrayList();
if (functions.containsKey(signature)) {
for (FunctionDescProto func : functions.get(signature)) {
if (func.getSignature().getParameterTypesList() != null
&& func.getSignature().getParameterTypesList().equals(params)) {
candidates.add(func);
}
}
}
/*
*
* FALL BACK to look for nearest match
* WORKING BUT BAD WAY TO IMPLEMENT.I WOULD RATHER implement compareTo in FunctionDesc to keep them
* in sorted order of param types LIKE INT1 SHOULD BE BEFORE INT2 should be before INT3 so on.
* Due to possibility of multiple parameters and types the permutation and combinations are endless
* to implement compareTo so decided to take the shortcut.
*
* */
if (functions.containsKey(signature)) {
for (FunctionDescProto func : functions.get(signature)) {
if (func.getSignature().getParameterTypesList() != null
&& CatalogUtil.isMatchedFunction(
func.getSignature().getParameterTypesList(), params)) {
candidates.add(func);
}
}
// if there are more than one function candidates, we choose the nearest matched function.
if (candidates.size() > 0) {
return findNearestMatchedFunction(candidates);
} else {
return null;
}
}
return null;
}
@Override
public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescProto)
throws CatalogException {
final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName());
if (split.length == 1) {
throw new IllegalArgumentException(
"alterTable() requires a qualified table name, but it is \""
+ alterTableDescProto.getTableName()
+ "\".");
}
final String databaseName = split[0];
final String tableName = split[1];
String partitionName = null;
CatalogProtos.PartitionDescProto partitionDesc = null;
switch (alterTableDescProto.getAlterTableType()) {
case RENAME_TABLE:
if (existTable(databaseName, alterTableDescProto.getNewTableName().toLowerCase())) {
throw new DuplicateTableException(alterTableDescProto.getNewTableName());
}
renameTable(databaseName, tableName, alterTableDescProto.getNewTableName().toLowerCase());
break;
case RENAME_COLUMN:
if (existColumn(
databaseName, tableName, alterTableDescProto.getAlterColumnName().getNewColumnName())) {
throw new DuplicateColumnException(
alterTableDescProto.getAlterColumnName().getNewColumnName());
}
renameColumn(databaseName, tableName, alterTableDescProto.getAlterColumnName());
break;
case ADD_COLUMN:
if (existColumn(databaseName, tableName, alterTableDescProto.getAddColumn().getName())) {
throw new DuplicateColumnException(alterTableDescProto.getAddColumn().getName());
}
addNewColumn(databaseName, tableName, alterTableDescProto.getAddColumn());
break;
case ADD_PARTITION:
partitionName = alterTableDescProto.getPartitionDesc().getPartitionName();
partitionDesc = getPartition(databaseName, tableName, partitionName);
if (partitionDesc != null) {
throw new DuplicatePartitionException(partitionName);
}
addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc());
break;
case DROP_PARTITION:
partitionName = alterTableDescProto.getPartitionDesc().getPartitionName();
partitionDesc = getPartition(databaseName, tableName, partitionName);
if (partitionDesc == null) {
throw new UndefinedPartitionException(partitionName);
}
dropPartition(databaseName, tableName, partitionDesc);
break;
case SET_PROPERTY:
// TODO - not implemented yet
break;
default:
// TODO
}
}
// TODO: This should be added at TAJO-1891
public final void testAddAndDeleteTablePartitionByColumn() throws Exception {
Schema schema =
SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
.add("age", Type.INT4)
.add("score", Type.FLOAT8)
.build();
String simpleTableName = "addedtable";
String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName);
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts);
Schema partSchema = SchemaBuilder.builder().add("id", Type.INT4).add("name", Type.TEXT).build();
PartitionMethodDesc partitionMethodDesc =
new PartitionMethodDesc(
DEFAULT_DATABASE_NAME,
tableName,
CatalogProtos.PartitionType.COLUMN,
"id,name",
partSchema);
TableDesc desc =
new TableDesc(
tableName,
schema,
meta,
new Path(CommonTestingUtil.getTestDir(), simpleTableName).toUri());
desc.setPartitionMethod(partitionMethodDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
assertTrue(catalog.existsTable(tableName));
TableDesc retrieved = catalog.getTableDesc(tableName);
assertEquals(retrieved.getName(), tableName);
assertEquals(
retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN);
assertEquals(
retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id");
testAddPartition(tableName, "id=10/name=aaa");
testAddPartition(tableName, "id=20/name=bbb");
List<CatalogProtos.PartitionDescProto> partitions =
catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 2);
assertEquals(partitions.get(0).getNumBytes(), 0L);
testGetPartitionsByAlgebra(DEFAULT_DATABASE_NAME, simpleTableName);
testDropPartition(tableName, "id=10/name=aaa");
testDropPartition(tableName, "id=20/name=bbb");
partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName);
assertNotNull(partitions);
assertEquals(partitions.size(), 0);
catalog.dropTable(tableName);
assertFalse(catalog.existsTable(tableName));
}
@Override
public final void createTable(final CatalogProtos.TableDescProto tableDescProto)
throws CatalogException {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
TableDesc tableDesc = new TableDesc(tableDescProto);
String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
String databaseName = splitted[0];
String tableName = splitted[1];
try {
client = clientPool.getClient();
org.apache.hadoop.hive.metastore.api.Table table =
new org.apache.hadoop.hive.metastore.api.Table();
table.setDbName(databaseName);
table.setTableName(tableName);
table.setParameters(
new HashMap<String, String>(tableDesc.getMeta().getOptions().getAllKeyValus()));
// TODO: set owner
// table.setOwner();
StorageDescriptor sd = new StorageDescriptor();
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().setName(table.getTableName());
// if tajo set location method, thrift client make exception as follows:
// Caused by: MetaException(message:java.lang.NullPointerException)
// If you want to modify table path, you have to modify on Hive cli.
if (tableDesc.isExternal()) {
table.setTableType(TableType.EXTERNAL_TABLE.name());
table.putToParameters("EXTERNAL", "TRUE");
Path tablePath = new Path(tableDesc.getUri());
FileSystem fs = tablePath.getFileSystem(conf);
if (fs.isFile(tablePath)) {
LOG.warn("A table path is a file, but HiveCatalogStore does not allow a file path.");
sd.setLocation(tablePath.getParent().toString());
} else {
sd.setLocation(tablePath.toString());
}
}
// set column information
List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
cols.add(
new FieldSchema(
eachField.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachField.getDataType()),
""));
}
sd.setCols(cols);
// set partition keys
if (tableDesc.hasPartition()
&& tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
for (Column eachPartitionKey :
tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(
new FieldSchema(
eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()),
""));
}
table.setPartitionKeys(partitionKeys);
}
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.TEXT_DELIMITER);
if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL)));
table.getParameters().remove(StorageConstants.TEXT_NULL);
}
} else if (tableDesc
.getMeta()
.getStoreType()
.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
String fieldDelimiter =
tableDesc
.getMeta()
.getOption(
StorageConstants.SEQUENCEFILE_DELIMITER,
StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
// And hive will un-espace this value again.
// As a result, user can use right field delimiter.
// So, we have to un-escape this value.
sd.getSerdeInfo()
.putToParameters(
serdeConstants.SERIALIZATION_FORMAT,
StringEscapeUtils.unescapeJava(fieldDelimiter));
sd.getSerdeInfo()
.putToParameters(
serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER);
} else {
sd.getSerdeInfo()
.setSerializationLib(
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) {
table.putToParameters(
serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(
tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
sd.getSerdeInfo()
.setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
} else {
throw new UnsupportedException(
tableDesc.getMeta().getStoreType() + " in HivecatalogStore");
}
}
sd.setSortCols(new ArrayList<Order>());
table.setSd(sd);
client.getHiveClient().createTable(table);
} catch (Throwable t) {
throw new TajoInternalError(t);
} finally {
if (client != null) client.release();
}
}