@Override
public void read(org.apache.thrift.protocol.TProtocol prot, TPrioritizeLoadRequest struct)
throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
struct.protocol_version = CatalogServiceVersion.findByValue(iprot.readI32());
struct.setProtocol_versionIsSet(true);
{
org.apache.thrift.protocol.TList _list21 =
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
struct.object_descs =
new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list21.size);
for (int _i22 = 0; _i22 < _list21.size; ++_i22) {
com.cloudera.impala.thrift.TCatalogObject _elem23; // required
_elem23 = new com.cloudera.impala.thrift.TCatalogObject();
_elem23.read(iprot);
struct.object_descs.add(_elem23);
}
}
struct.setObject_descsIsSet(true);
BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
struct.header = new TCatalogServiceRequestHeader();
struct.header.read(iprot);
struct.setHeaderIsSet(true);
}
}
public void write(org.apache.thrift.protocol.TProtocol oprot, TPrioritizeLoadRequest struct)
throws org.apache.thrift.TException {
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
if (struct.protocol_version != null) {
oprot.writeFieldBegin(PROTOCOL_VERSION_FIELD_DESC);
oprot.writeI32(struct.protocol_version.getValue());
oprot.writeFieldEnd();
}
if (struct.header != null) {
if (struct.isSetHeader()) {
oprot.writeFieldBegin(HEADER_FIELD_DESC);
struct.header.write(oprot);
oprot.writeFieldEnd();
}
}
if (struct.object_descs != null) {
oprot.writeFieldBegin(OBJECT_DESCS_FIELD_DESC);
{
oprot.writeListBegin(
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, struct.object_descs.size()));
for (com.cloudera.impala.thrift.TCatalogObject _iter19 : struct.object_descs) {
_iter19.write(oprot);
}
oprot.writeListEnd();
}
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
public void write(
org.apache.thrift.protocol.TProtocol oprot, TGetAllCatalogObjectsResponse struct)
throws org.apache.thrift.TException {
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
oprot.writeFieldBegin(MAX_CATALOG_VERSION_FIELD_DESC);
oprot.writeI64(struct.max_catalog_version);
oprot.writeFieldEnd();
if (struct.objects != null) {
oprot.writeFieldBegin(OBJECTS_FIELD_DESC);
{
oprot.writeListBegin(
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, struct.objects.size()));
for (com.cloudera.impala.thrift.TCatalogObject _iter3 : struct.objects) {
_iter3.write(oprot);
}
oprot.writeListEnd();
}
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
/**
* Prioritizes the loading of the given list TCatalogObjects. Currently only support loading
* Table/View metadata since Db and Function metadata is not loaded lazily.
*/
public void prioritizeLoad(List<TCatalogObject> objectDescs) {
for (TCatalogObject catalogObject : objectDescs) {
Preconditions.checkState(catalogObject.isSetTable());
TTable table = catalogObject.getTable();
tableLoadingMgr_.prioritizeLoad(
new TTableName(table.getDb_name().toLowerCase(), table.getTbl_name().toLowerCase()));
}
}
public void read(org.apache.thrift.protocol.TProtocol iprot, TPrioritizeLoadRequest struct)
throws org.apache.thrift.TException {
org.apache.thrift.protocol.TField schemeField;
iprot.readStructBegin();
while (true) {
schemeField = iprot.readFieldBegin();
if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
case 1: // PROTOCOL_VERSION
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
struct.protocol_version = CatalogServiceVersion.findByValue(iprot.readI32());
struct.setProtocol_versionIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 2: // HEADER
if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
struct.header = new TCatalogServiceRequestHeader();
struct.header.read(iprot);
struct.setHeaderIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 3: // OBJECT_DESCS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
org.apache.thrift.protocol.TList _list16 = iprot.readListBegin();
struct.object_descs =
new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list16.size);
for (int _i17 = 0; _i17 < _list16.size; ++_i17) {
com.cloudera.impala.thrift.TCatalogObject _elem18; // required
_elem18 = new com.cloudera.impala.thrift.TCatalogObject();
_elem18.read(iprot);
struct.object_descs.add(_elem18);
}
iprot.readListEnd();
}
struct.setObject_descsIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
iprot.readFieldEnd();
}
iprot.readStructEnd();
// check for required fields of primitive type, which can't be checked in the validate method
struct.validate();
}
public void read(
org.apache.thrift.protocol.TProtocol iprot, TGetAllCatalogObjectsResponse struct)
throws org.apache.thrift.TException {
org.apache.thrift.protocol.TField schemeField;
iprot.readStructBegin();
while (true) {
schemeField = iprot.readFieldBegin();
if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
case 1: // MAX_CATALOG_VERSION
if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
struct.max_catalog_version = iprot.readI64();
struct.setMax_catalog_versionIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 2: // OBJECTS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
struct.objects =
new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list0.size);
for (int _i1 = 0; _i1 < _list0.size; ++_i1) {
com.cloudera.impala.thrift.TCatalogObject _elem2; // required
_elem2 = new com.cloudera.impala.thrift.TCatalogObject();
_elem2.read(iprot);
struct.objects.add(_elem2);
}
iprot.readListEnd();
}
struct.setObjectsIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
iprot.readFieldEnd();
}
iprot.readStructEnd();
// check for required fields of primitive type, which can't be checked in the validate method
if (!struct.isSetMax_catalog_version()) {
throw new org.apache.thrift.protocol.TProtocolException(
"Required field 'max_catalog_version' was not found in serialized data! Struct: "
+ toString());
}
struct.validate();
}
@Override
public void write(
org.apache.thrift.protocol.TProtocol prot, TGetAllCatalogObjectsResponse struct)
throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
oprot.writeI64(struct.max_catalog_version);
{
oprot.writeI32(struct.objects.size());
for (com.cloudera.impala.thrift.TCatalogObject _iter4 : struct.objects) {
_iter4.write(oprot);
}
}
}
@Override
public void write(org.apache.thrift.protocol.TProtocol prot, TPrioritizeLoadRequest struct)
throws org.apache.thrift.TException {
TTupleProtocol oprot = (TTupleProtocol) prot;
oprot.writeI32(struct.protocol_version.getValue());
{
oprot.writeI32(struct.object_descs.size());
for (com.cloudera.impala.thrift.TCatalogObject _iter20 : struct.object_descs) {
_iter20.write(oprot);
}
}
BitSet optionals = new BitSet();
if (struct.isSetHeader()) {
optionals.set(0);
}
oprot.writeBitSet(optionals, 1);
if (struct.isSetHeader()) {
struct.header.write(oprot);
}
}
@Override
public void read(
org.apache.thrift.protocol.TProtocol prot, TGetAllCatalogObjectsResponse struct)
throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
struct.max_catalog_version = iprot.readI64();
struct.setMax_catalog_versionIsSet(true);
{
org.apache.thrift.protocol.TList _list5 =
new org.apache.thrift.protocol.TList(
org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
struct.objects = new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list5.size);
for (int _i6 = 0; _i6 < _list5.size; ++_i6) {
com.cloudera.impala.thrift.TCatalogObject _elem7; // required
_elem7 = new com.cloudera.impala.thrift.TCatalogObject();
_elem7.read(iprot);
struct.objects.add(_elem7);
}
}
struct.setObjectsIsSet(true);
}
/**
* Returns all known objects in the Catalog (Tables, Views, Databases, and Functions). Some
* metadata may be skipped for objects that have a catalog version < the specified "fromVersion".
* Skips builtins.
*/
public TGetAllCatalogObjectsResponse getCatalogObjects(long fromVersion) {
TGetAllCatalogObjectsResponse resp = new TGetAllCatalogObjectsResponse();
resp.setObjects(new ArrayList<TCatalogObject>());
resp.setMax_catalog_version(Catalog.INITIAL_CATALOG_VERSION);
// Take a lock on the catalog to ensure this update contains a consistent snapshot
// of all items in the catalog.
catalogLock_.readLock().lock();
try {
for (String dbName : getDbNames(null)) {
Db db = getDb(dbName);
if (db == null) {
LOG.error(
"Database: "
+ dbName
+ " was expected to be in the catalog "
+ "cache. Skipping database and all child objects for this update.");
continue;
}
if (db.isSystemDb()) continue;
TCatalogObject catalogDb =
new TCatalogObject(TCatalogObjectType.DATABASE, db.getCatalogVersion());
catalogDb.setDb(db.toThrift());
resp.addToObjects(catalogDb);
for (String tblName : db.getAllTableNames()) {
TCatalogObject catalogTbl =
new TCatalogObject(TCatalogObjectType.TABLE, Catalog.INITIAL_CATALOG_VERSION);
Table tbl = db.getTable(tblName);
if (tbl == null) {
LOG.error(
"Table: "
+ tblName
+ " was expected to be in the catalog "
+ "cache. Skipping table for this update.");
continue;
}
// Only add the extended metadata if this table's version is >=
// the fromVersion.
if (tbl.getCatalogVersion() >= fromVersion) {
try {
catalogTbl.setTable(tbl.toThrift());
} catch (Exception e) {
LOG.debug(
String.format(
"Error calling toThrift() on table %s.%s: %s",
dbName, tblName, e.getMessage()),
e);
continue;
}
catalogTbl.setCatalog_version(tbl.getCatalogVersion());
} else {
catalogTbl.setTable(new TTable(dbName, tblName));
}
resp.addToObjects(catalogTbl);
}
for (Function fn : db.getFunctions(null, new PatternMatcher())) {
TCatalogObject function =
new TCatalogObject(TCatalogObjectType.FUNCTION, fn.getCatalogVersion());
function.setFn(fn.toThrift());
resp.addToObjects(function);
}
}
for (DataSource dataSource : getDataSources()) {
TCatalogObject catalogObj =
new TCatalogObject(TCatalogObjectType.DATA_SOURCE, dataSource.getCatalogVersion());
catalogObj.setData_source(dataSource.toThrift());
resp.addToObjects(catalogObj);
}
for (HdfsCachePool cachePool : hdfsCachePools_) {
TCatalogObject pool =
new TCatalogObject(TCatalogObjectType.HDFS_CACHE_POOL, cachePool.getCatalogVersion());
pool.setCache_pool(cachePool.toThrift());
resp.addToObjects(pool);
}
// Get all roles
for (Role role : authPolicy_.getAllRoles()) {
TCatalogObject thriftRole = new TCatalogObject();
thriftRole.setRole(role.toThrift());
thriftRole.setCatalog_version(role.getCatalogVersion());
thriftRole.setType(role.getCatalogObjectType());
resp.addToObjects(thriftRole);
for (RolePrivilege p : role.getPrivileges()) {
TCatalogObject privilege = new TCatalogObject();
privilege.setPrivilege(p.toThrift());
privilege.setCatalog_version(p.getCatalogVersion());
privilege.setType(p.getCatalogObjectType());
resp.addToObjects(privilege);
}
}
// Each update should contain a single "TCatalog" object which is used to
// pass overall state on the catalog, such as the current version and the
// catalog service id.
TCatalogObject catalog = new TCatalogObject();
catalog.setType(TCatalogObjectType.CATALOG);
// By setting the catalog version to the latest catalog version at this point,
// it ensure impalads will always bump their versions, even in the case where
// an object has been dropped.
catalog.setCatalog_version(getCatalogVersion());
catalog.setCatalog(new TCatalog(catalogServiceId_));
resp.addToObjects(catalog);
// The max version is the max catalog version of all items in the update.
resp.setMax_catalog_version(getCatalogVersion());
return resp;
} finally {
catalogLock_.readLock().unlock();
}
}
