コード例 #1
0
 @Override
 public void read(org.apache.thrift.protocol.TProtocol prot, TPrioritizeLoadRequest struct)
     throws org.apache.thrift.TException {
   TTupleProtocol iprot = (TTupleProtocol) prot;
   struct.protocol_version = CatalogServiceVersion.findByValue(iprot.readI32());
   struct.setProtocol_versionIsSet(true);
   {
     org.apache.thrift.protocol.TList _list21 =
         new org.apache.thrift.protocol.TList(
             org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
     struct.object_descs =
         new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list21.size);
     for (int _i22 = 0; _i22 < _list21.size; ++_i22) {
       com.cloudera.impala.thrift.TCatalogObject _elem23; // required
       _elem23 = new com.cloudera.impala.thrift.TCatalogObject();
       _elem23.read(iprot);
       struct.object_descs.add(_elem23);
     }
   }
   struct.setObject_descsIsSet(true);
   BitSet incoming = iprot.readBitSet(1);
   if (incoming.get(0)) {
     struct.header = new TCatalogServiceRequestHeader();
     struct.header.read(iprot);
     struct.setHeaderIsSet(true);
   }
 }
コード例 #2
0
    public void write(org.apache.thrift.protocol.TProtocol oprot, TPrioritizeLoadRequest struct)
        throws org.apache.thrift.TException {
      struct.validate();

      oprot.writeStructBegin(STRUCT_DESC);
      if (struct.protocol_version != null) {
        oprot.writeFieldBegin(PROTOCOL_VERSION_FIELD_DESC);
        oprot.writeI32(struct.protocol_version.getValue());
        oprot.writeFieldEnd();
      }
      if (struct.header != null) {
        if (struct.isSetHeader()) {
          oprot.writeFieldBegin(HEADER_FIELD_DESC);
          struct.header.write(oprot);
          oprot.writeFieldEnd();
        }
      }
      if (struct.object_descs != null) {
        oprot.writeFieldBegin(OBJECT_DESCS_FIELD_DESC);
        {
          oprot.writeListBegin(
              new org.apache.thrift.protocol.TList(
                  org.apache.thrift.protocol.TType.STRUCT, struct.object_descs.size()));
          for (com.cloudera.impala.thrift.TCatalogObject _iter19 : struct.object_descs) {
            _iter19.write(oprot);
          }
          oprot.writeListEnd();
        }
        oprot.writeFieldEnd();
      }
      oprot.writeFieldStop();
      oprot.writeStructEnd();
    }
コード例 #3
0
    public void write(
        org.apache.thrift.protocol.TProtocol oprot, TGetAllCatalogObjectsResponse struct)
        throws org.apache.thrift.TException {
      struct.validate();

      oprot.writeStructBegin(STRUCT_DESC);
      oprot.writeFieldBegin(MAX_CATALOG_VERSION_FIELD_DESC);
      oprot.writeI64(struct.max_catalog_version);
      oprot.writeFieldEnd();
      if (struct.objects != null) {
        oprot.writeFieldBegin(OBJECTS_FIELD_DESC);
        {
          oprot.writeListBegin(
              new org.apache.thrift.protocol.TList(
                  org.apache.thrift.protocol.TType.STRUCT, struct.objects.size()));
          for (com.cloudera.impala.thrift.TCatalogObject _iter3 : struct.objects) {
            _iter3.write(oprot);
          }
          oprot.writeListEnd();
        }
        oprot.writeFieldEnd();
      }
      oprot.writeFieldStop();
      oprot.writeStructEnd();
    }
コード例 #4
0
 /**
  * Prioritizes the loading of the given list TCatalogObjects. Currently only support loading
  * Table/View metadata since Db and Function metadata is not loaded lazily.
  */
 public void prioritizeLoad(List<TCatalogObject> objectDescs) {
   for (TCatalogObject catalogObject : objectDescs) {
     Preconditions.checkState(catalogObject.isSetTable());
     TTable table = catalogObject.getTable();
     tableLoadingMgr_.prioritizeLoad(
         new TTableName(table.getDb_name().toLowerCase(), table.getTbl_name().toLowerCase()));
   }
 }
コード例 #5
0
    public void read(org.apache.thrift.protocol.TProtocol iprot, TPrioritizeLoadRequest struct)
        throws org.apache.thrift.TException {
      org.apache.thrift.protocol.TField schemeField;
      iprot.readStructBegin();
      while (true) {
        schemeField = iprot.readFieldBegin();
        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
          break;
        }
        switch (schemeField.id) {
          case 1: // PROTOCOL_VERSION
            if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
              struct.protocol_version = CatalogServiceVersion.findByValue(iprot.readI32());
              struct.setProtocol_versionIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          case 2: // HEADER
            if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
              struct.header = new TCatalogServiceRequestHeader();
              struct.header.read(iprot);
              struct.setHeaderIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          case 3: // OBJECT_DESCS
            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
              {
                org.apache.thrift.protocol.TList _list16 = iprot.readListBegin();
                struct.object_descs =
                    new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list16.size);
                for (int _i17 = 0; _i17 < _list16.size; ++_i17) {
                  com.cloudera.impala.thrift.TCatalogObject _elem18; // required
                  _elem18 = new com.cloudera.impala.thrift.TCatalogObject();
                  _elem18.read(iprot);
                  struct.object_descs.add(_elem18);
                }
                iprot.readListEnd();
              }
              struct.setObject_descsIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          default:
            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
        }
        iprot.readFieldEnd();
      }
      iprot.readStructEnd();

      // check for required fields of primitive type, which can't be checked in the validate method
      struct.validate();
    }
コード例 #6
0
    public void read(
        org.apache.thrift.protocol.TProtocol iprot, TGetAllCatalogObjectsResponse struct)
        throws org.apache.thrift.TException {
      org.apache.thrift.protocol.TField schemeField;
      iprot.readStructBegin();
      while (true) {
        schemeField = iprot.readFieldBegin();
        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
          break;
        }
        switch (schemeField.id) {
          case 1: // MAX_CATALOG_VERSION
            if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
              struct.max_catalog_version = iprot.readI64();
              struct.setMax_catalog_versionIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          case 2: // OBJECTS
            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
              {
                org.apache.thrift.protocol.TList _list0 = iprot.readListBegin();
                struct.objects =
                    new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list0.size);
                for (int _i1 = 0; _i1 < _list0.size; ++_i1) {
                  com.cloudera.impala.thrift.TCatalogObject _elem2; // required
                  _elem2 = new com.cloudera.impala.thrift.TCatalogObject();
                  _elem2.read(iprot);
                  struct.objects.add(_elem2);
                }
                iprot.readListEnd();
              }
              struct.setObjectsIsSet(true);
            } else {
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
            }
            break;
          default:
            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
        }
        iprot.readFieldEnd();
      }
      iprot.readStructEnd();

      // check for required fields of primitive type, which can't be checked in the validate method
      if (!struct.isSetMax_catalog_version()) {
        throw new org.apache.thrift.protocol.TProtocolException(
            "Required field 'max_catalog_version' was not found in serialized data! Struct: "
                + toString());
      }
      struct.validate();
    }
コード例 #7
0
 @Override
 public void write(
     org.apache.thrift.protocol.TProtocol prot, TGetAllCatalogObjectsResponse struct)
     throws org.apache.thrift.TException {
   TTupleProtocol oprot = (TTupleProtocol) prot;
   oprot.writeI64(struct.max_catalog_version);
   {
     oprot.writeI32(struct.objects.size());
     for (com.cloudera.impala.thrift.TCatalogObject _iter4 : struct.objects) {
       _iter4.write(oprot);
     }
   }
 }
コード例 #8
0
 @Override
 public void write(org.apache.thrift.protocol.TProtocol prot, TPrioritizeLoadRequest struct)
     throws org.apache.thrift.TException {
   TTupleProtocol oprot = (TTupleProtocol) prot;
   oprot.writeI32(struct.protocol_version.getValue());
   {
     oprot.writeI32(struct.object_descs.size());
     for (com.cloudera.impala.thrift.TCatalogObject _iter20 : struct.object_descs) {
       _iter20.write(oprot);
     }
   }
   BitSet optionals = new BitSet();
   if (struct.isSetHeader()) {
     optionals.set(0);
   }
   oprot.writeBitSet(optionals, 1);
   if (struct.isSetHeader()) {
     struct.header.write(oprot);
   }
 }
コード例 #9
0
 @Override
 public void read(
     org.apache.thrift.protocol.TProtocol prot, TGetAllCatalogObjectsResponse struct)
     throws org.apache.thrift.TException {
   TTupleProtocol iprot = (TTupleProtocol) prot;
   struct.max_catalog_version = iprot.readI64();
   struct.setMax_catalog_versionIsSet(true);
   {
     org.apache.thrift.protocol.TList _list5 =
         new org.apache.thrift.protocol.TList(
             org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
     struct.objects = new ArrayList<com.cloudera.impala.thrift.TCatalogObject>(_list5.size);
     for (int _i6 = 0; _i6 < _list5.size; ++_i6) {
       com.cloudera.impala.thrift.TCatalogObject _elem7; // required
       _elem7 = new com.cloudera.impala.thrift.TCatalogObject();
       _elem7.read(iprot);
       struct.objects.add(_elem7);
     }
   }
   struct.setObjectsIsSet(true);
 }
コード例 #10
0
  /**
   * Returns all known objects in the Catalog (Tables, Views, Databases, and Functions). Some
   * metadata may be skipped for objects that have a catalog version < the specified "fromVersion".
   * Skips builtins.
   */
  public TGetAllCatalogObjectsResponse getCatalogObjects(long fromVersion) {
    TGetAllCatalogObjectsResponse resp = new TGetAllCatalogObjectsResponse();
    resp.setObjects(new ArrayList<TCatalogObject>());
    resp.setMax_catalog_version(Catalog.INITIAL_CATALOG_VERSION);

    // Take a lock on the catalog to ensure this update contains a consistent snapshot
    // of all items in the catalog.
    catalogLock_.readLock().lock();
    try {
      for (String dbName : getDbNames(null)) {
        Db db = getDb(dbName);
        if (db == null) {
          LOG.error(
              "Database: "
                  + dbName
                  + " was expected to be in the catalog "
                  + "cache. Skipping database and all child objects for this update.");
          continue;
        }
        if (db.isSystemDb()) continue;
        TCatalogObject catalogDb =
            new TCatalogObject(TCatalogObjectType.DATABASE, db.getCatalogVersion());
        catalogDb.setDb(db.toThrift());
        resp.addToObjects(catalogDb);

        for (String tblName : db.getAllTableNames()) {
          TCatalogObject catalogTbl =
              new TCatalogObject(TCatalogObjectType.TABLE, Catalog.INITIAL_CATALOG_VERSION);

          Table tbl = db.getTable(tblName);
          if (tbl == null) {
            LOG.error(
                "Table: "
                    + tblName
                    + " was expected to be in the catalog "
                    + "cache. Skipping table for this update.");
            continue;
          }

          // Only add the extended metadata if this table's version is >=
          // the fromVersion.
          if (tbl.getCatalogVersion() >= fromVersion) {
            try {
              catalogTbl.setTable(tbl.toThrift());
            } catch (Exception e) {
              LOG.debug(
                  String.format(
                      "Error calling toThrift() on table %s.%s: %s",
                      dbName, tblName, e.getMessage()),
                  e);
              continue;
            }
            catalogTbl.setCatalog_version(tbl.getCatalogVersion());
          } else {
            catalogTbl.setTable(new TTable(dbName, tblName));
          }
          resp.addToObjects(catalogTbl);
        }

        for (Function fn : db.getFunctions(null, new PatternMatcher())) {
          TCatalogObject function =
              new TCatalogObject(TCatalogObjectType.FUNCTION, fn.getCatalogVersion());
          function.setFn(fn.toThrift());
          resp.addToObjects(function);
        }
      }

      for (DataSource dataSource : getDataSources()) {
        TCatalogObject catalogObj =
            new TCatalogObject(TCatalogObjectType.DATA_SOURCE, dataSource.getCatalogVersion());
        catalogObj.setData_source(dataSource.toThrift());
        resp.addToObjects(catalogObj);
      }
      for (HdfsCachePool cachePool : hdfsCachePools_) {
        TCatalogObject pool =
            new TCatalogObject(TCatalogObjectType.HDFS_CACHE_POOL, cachePool.getCatalogVersion());
        pool.setCache_pool(cachePool.toThrift());
        resp.addToObjects(pool);
      }

      // Get all roles
      for (Role role : authPolicy_.getAllRoles()) {
        TCatalogObject thriftRole = new TCatalogObject();
        thriftRole.setRole(role.toThrift());
        thriftRole.setCatalog_version(role.getCatalogVersion());
        thriftRole.setType(role.getCatalogObjectType());
        resp.addToObjects(thriftRole);

        for (RolePrivilege p : role.getPrivileges()) {
          TCatalogObject privilege = new TCatalogObject();
          privilege.setPrivilege(p.toThrift());
          privilege.setCatalog_version(p.getCatalogVersion());
          privilege.setType(p.getCatalogObjectType());
          resp.addToObjects(privilege);
        }
      }

      // Each update should contain a single "TCatalog" object which is used to
      // pass overall state on the catalog, such as the current version and the
      // catalog service id.
      TCatalogObject catalog = new TCatalogObject();
      catalog.setType(TCatalogObjectType.CATALOG);
      // By setting the catalog version to the latest catalog version at this point,
      // it ensure impalads will always bump their versions, even in the case where
      // an object has been dropped.
      catalog.setCatalog_version(getCatalogVersion());
      catalog.setCatalog(new TCatalog(catalogServiceId_));
      resp.addToObjects(catalog);

      // The max version is the max catalog version of all items in the update.
      resp.setMax_catalog_version(getCatalogVersion());
      return resp;
    } finally {
      catalogLock_.readLock().unlock();
    }
  }