コード例 #1
0
 /**
  * Updates the cached lastDdlTime for the given table. The lastDdlTime is used during the metadata
  * refresh() operations to determine if there have been any external (outside of Impala)
  * modifications to the table.
  */
 public void updateLastDdlTime(TTableName tblName, long ddlTime) {
   Db db = getDb(tblName.getDb_name());
   if (db == null) return;
   Table tbl = db.getTable(tblName.getTable_name());
   if (tbl == null) return;
   tbl.updateLastDdlTime(ddlTime);
 }
コード例 #2
0
 /**
  * Adds a table with the given name to the catalog and returns the new table, loading the metadata
  * if needed.
  */
 public Table addTable(String dbName, String tblName) throws TableNotFoundException {
   Db db = getDb(dbName);
   if (db == null) return null;
   Table incompleteTable = IncompleteTable.createUninitializedTable(getNextTableId(), db, tblName);
   incompleteTable.setCatalogVersion(incrementAndGetCatalogVersion());
   db.addTable(incompleteTable);
   return db.getTable(tblName);
 }
コード例 #3
0
ファイル: CatalogTest.java プロジェクト: hihihippp/Impala
 private void checkTableCols(
     Db db, String tblName, int numClusteringCols, String[] colNames, ColumnType[] colTypes)
     throws TableLoadingException {
   Table tbl = db.getTable(tblName);
   assertEquals(tbl.getName(), tblName);
   assertEquals(tbl.getNumClusteringCols(), numClusteringCols);
   List<Column> cols = tbl.getColumns();
   assertEquals(colNames.length, colTypes.length);
   assertEquals(cols.size(), colNames.length);
   Iterator<Column> it = cols.iterator();
   int i = 0;
   while (it.hasNext()) {
     Column col = it.next();
     assertEquals(col.getName(), colNames[i]);
     assertTrue(col.getType().equals(colTypes[i]));
     ++i;
   }
 }
コード例 #4
0
  /**
   * Replaces an existing Table with a new value if it exists and has not changed (has the same
   * catalog version as 'expectedCatalogVersion').
   */
  private Table replaceTableIfUnchanged(Table updatedTbl, long expectedCatalogVersion)
      throws DatabaseNotFoundException {
    catalogLock_.writeLock().lock();
    try {
      Db db = getDb(updatedTbl.getDb().getName());
      if (db == null) {
        throw new DatabaseNotFoundException(
            "Database does not exist: " + updatedTbl.getDb().getName());
      }

      Table existingTbl = db.getTable(updatedTbl.getName());
      // The existing table does not exist or has been modified. Instead of
      // adding the loaded value, return the existing table.
      if (existingTbl == null || existingTbl.getCatalogVersion() != expectedCatalogVersion)
        return existingTbl;

      updatedTbl.setCatalogVersion(incrementAndGetCatalogVersion());
      db.addTable(updatedTbl);
      return updatedTbl;
    } finally {
      catalogLock_.writeLock().unlock();
    }
  }
コード例 #5
0
ファイル: CatalogTest.java プロジェクト: hihihippp/Impala
 private void checkHBaseTableCols(
     Db db,
     String hiveTableName,
     String hbaseTableName,
     String[] hiveColNames,
     String[] colFamilies,
     String[] colQualifiers,
     ColumnType[] colTypes)
     throws TableLoadingException {
   checkTableCols(db, hiveTableName, 1, hiveColNames, colTypes);
   HBaseTable tbl = (HBaseTable) db.getTable(hiveTableName);
   assertEquals(tbl.getHBaseTableName(), hbaseTableName);
   List<Column> cols = tbl.getColumns();
   assertEquals(colFamilies.length, colTypes.length);
   assertEquals(colQualifiers.length, colTypes.length);
   Iterator<Column> it = cols.iterator();
   int i = 0;
   while (it.hasNext()) {
     HBaseColumn col = (HBaseColumn) it.next();
     assertEquals(col.getColumnFamily(), colFamilies[i]);
     assertEquals(col.getColumnQualifier(), colQualifiers[i]);
     ++i;
   }
 }
コード例 #6
0
ファイル: CatalogTest.java プロジェクト: hihihippp/Impala
  @Test
  public void TestColSchema() throws TableLoadingException {
    Db functionalDb = catalog_.getDb("functional");
    assertNotNull(functionalDb);
    assertEquals(functionalDb.getName(), "functional");
    assertNotNull(functionalDb.getTable("alltypes"));
    assertNotNull(functionalDb.getTable("alltypes_view"));
    assertNotNull(functionalDb.getTable("alltypes_view_sub"));
    assertNotNull(functionalDb.getTable("alltypessmall"));
    assertNotNull(functionalDb.getTable("alltypeserror"));
    assertNotNull(functionalDb.getTable("alltypeserrornonulls"));
    assertNotNull(functionalDb.getTable("alltypesagg"));
    assertNotNull(functionalDb.getTable("alltypesaggnonulls"));
    assertNotNull(functionalDb.getTable("alltypesnopart"));
    assertNotNull(functionalDb.getTable("alltypesinsert"));
    assertNotNull(functionalDb.getTable("complex_view"));
    assertNotNull(functionalDb.getTable("testtbl"));
    assertNotNull(functionalDb.getTable("dimtbl"));
    assertNotNull(functionalDb.getTable("jointbl"));
    assertNotNull(functionalDb.getTable("liketbl"));
    assertNotNull(functionalDb.getTable("greptiny"));
    assertNotNull(functionalDb.getTable("rankingssmall"));
    assertNotNull(functionalDb.getTable("uservisitssmall"));
    assertNotNull(functionalDb.getTable("view_view"));
    // IMP-163 - table with string partition column does not load if there are partitions
    assertNotNull(functionalDb.getTable("StringPartitionKey"));
    // Test non-existent table
    assertNull(functionalDb.getTable("nonexistenttable"));

    // functional_seq contains the same tables as functional
    Db testDb = catalog_.getDb("functional_seq");
    assertNotNull(testDb);
    assertEquals(testDb.getName(), "functional_seq");
    assertNotNull(testDb.getTable("alltypes"));
    assertNotNull(testDb.getTable("testtbl"));

    Db hbaseDb = catalog_.getDb("functional_hbase");
    assertNotNull(hbaseDb);
    assertEquals(hbaseDb.getName(), "functional_hbase");
    // Loading succeeds for an HBase table that has binary columns and an implicit key
    // column mapping
    assertNotNull(hbaseDb.getTable("alltypessmallbinary"));
    assertNotNull(hbaseDb.getTable("alltypessmall"));
    assertNotNull(hbaseDb.getTable("hbasealltypeserror"));
    assertNotNull(hbaseDb.getTable("hbasealltypeserrornonulls"));
    assertNotNull(hbaseDb.getTable("alltypesagg"));
    assertNotNull(hbaseDb.getTable("stringids"));

    checkTableCols(
        functionalDb,
        "alltypes",
        2,
        new String[] {
          "year",
          "month",
          "id",
          "bool_col",
          "tinyint_col",
          "smallint_col",
          "int_col",
          "bigint_col",
          "float_col",
          "double_col",
          "date_string_col",
          "string_col",
          "timestamp_col"
        },
        new ColumnType[] {
          ColumnType.INT, ColumnType.INT, ColumnType.INT,
          ColumnType.BOOLEAN, ColumnType.TINYINT, ColumnType.SMALLINT,
          ColumnType.INT, ColumnType.BIGINT, ColumnType.FLOAT,
          ColumnType.DOUBLE, ColumnType.STRING, ColumnType.STRING,
          ColumnType.TIMESTAMP
        });
    checkTableCols(
        functionalDb,
        "testtbl",
        0,
        new String[] {"id", "name", "zip"},
        new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
    checkTableCols(
        testDb,
        "testtbl",
        0,
        new String[] {"id", "name", "zip"},
        new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
    checkTableCols(
        functionalDb,
        "liketbl",
        0,
        new String[] {
          "str_col", "match_like_col", "no_match_like_col", "match_regex_col", "no_match_regex_col"
        },
        new ColumnType[] {
          ColumnType.STRING,
          ColumnType.STRING,
          ColumnType.STRING,
          ColumnType.STRING,
          ColumnType.STRING
        });
    checkTableCols(
        functionalDb,
        "dimtbl",
        0,
        new String[] {"id", "name", "zip"},
        new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT});
    checkTableCols(
        functionalDb,
        "jointbl",
        0,
        new String[] {"test_id", "test_name", "test_zip", "alltypes_id"},
        new ColumnType[] {ColumnType.BIGINT, ColumnType.STRING, ColumnType.INT, ColumnType.INT});

    checkHBaseTableCols(
        hbaseDb,
        "alltypessmall",
        "functional_hbase.alltypessmall",
        new String[] {
          "id",
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[] {
          null,
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new ColumnType[] {
          ColumnType.INT,
          ColumnType.BIGINT,
          ColumnType.BOOLEAN,
          ColumnType.STRING,
          ColumnType.DOUBLE,
          ColumnType.FLOAT,
          ColumnType.INT,
          ColumnType.INT,
          ColumnType.SMALLINT,
          ColumnType.STRING,
          ColumnType.TIMESTAMP,
          ColumnType.TINYINT,
          ColumnType.INT
        });

    checkHBaseTableCols(
        hbaseDb,
        "hbasealltypeserror",
        "functional_hbase.hbasealltypeserror",
        new String[] {
          "id",
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col"
        },
        new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[] {
          null,
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col"
        },
        new ColumnType[] {
          ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
          ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
          ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
          ColumnType.TIMESTAMP, ColumnType.TINYINT
        });

    checkHBaseTableCols(
        hbaseDb,
        "hbasealltypeserrornonulls",
        "functional_hbase.hbasealltypeserrornonulls",
        new String[] {
          "id",
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col"
        },
        new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[] {
          null,
          "bigint_col",
          "bool_col",
          "date_string_col",
          "double_col",
          "float_col",
          "int_col",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col"
        },
        new ColumnType[] {
          ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
          ColumnType.STRING, ColumnType.DOUBLE, ColumnType.FLOAT,
          ColumnType.INT, ColumnType.SMALLINT, ColumnType.STRING,
          ColumnType.TIMESTAMP, ColumnType.TINYINT
        });

    checkHBaseTableCols(
        hbaseDb,
        "alltypesagg",
        "functional_hbase.alltypesagg",
        new String[] {
          "id",
          "bigint_col",
          "bool_col",
          "date_string_col",
          "day",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[] {
          null,
          "bigint_col",
          "bool_col",
          "date_string_col",
          "day",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new ColumnType[] {
          ColumnType.INT, ColumnType.BIGINT, ColumnType.BOOLEAN,
          ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
          ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
          ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
          ColumnType.TINYINT, ColumnType.INT
        });

    checkHBaseTableCols(
        hbaseDb,
        "stringids",
        "functional_hbase.alltypesagg",
        new String[] {
          "id",
          "bigint_col",
          "bool_col",
          "date_string_col",
          "day",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new String[] {":key", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d", "d"},
        new String[] {
          null,
          "bigint_col",
          "bool_col",
          "date_string_col",
          "day",
          "double_col",
          "float_col",
          "int_col",
          "month",
          "smallint_col",
          "string_col",
          "timestamp_col",
          "tinyint_col",
          "year"
        },
        new ColumnType[] {
          ColumnType.STRING, ColumnType.BIGINT, ColumnType.BOOLEAN,
          ColumnType.STRING, ColumnType.INT, ColumnType.DOUBLE,
          ColumnType.FLOAT, ColumnType.INT, ColumnType.INT,
          ColumnType.SMALLINT, ColumnType.STRING, ColumnType.TIMESTAMP,
          ColumnType.TINYINT, ColumnType.INT
        });

    checkTableCols(
        functionalDb, "greptiny", 0, new String[] {"field"}, new ColumnType[] {ColumnType.STRING});

    checkTableCols(
        functionalDb,
        "rankingssmall",
        0,
        new String[] {"pagerank", "pageurl", "avgduration"},
        new ColumnType[] {ColumnType.INT, ColumnType.STRING, ColumnType.INT});

    checkTableCols(
        functionalDb,
        "uservisitssmall",
        0,
        new String[] {
          "sourceip",
          "desturl",
          "visitdate",
          "adrevenue",
          "useragent",
          "ccode",
          "lcode",
          "skeyword",
          "avgtimeonsite"
        },
        new ColumnType[] {
          ColumnType.STRING, ColumnType.STRING, ColumnType.STRING,
          ColumnType.FLOAT, ColumnType.STRING, ColumnType.STRING,
          ColumnType.STRING, ColumnType.STRING, ColumnType.INT
        });

    // case-insensitive lookup
    assertEquals(functionalDb.getTable("alltypes"), functionalDb.getTable("AllTypes"));
  }
コード例 #7
0
  /**
   * Returns all known objects in the Catalog (Tables, Views, Databases, and Functions). Some
   * metadata may be skipped for objects that have a catalog version < the specified "fromVersion".
   * Skips builtins.
   */
  public TGetAllCatalogObjectsResponse getCatalogObjects(long fromVersion) {
    TGetAllCatalogObjectsResponse resp = new TGetAllCatalogObjectsResponse();
    resp.setObjects(new ArrayList<TCatalogObject>());
    resp.setMax_catalog_version(Catalog.INITIAL_CATALOG_VERSION);

    // Take a lock on the catalog to ensure this update contains a consistent snapshot
    // of all items in the catalog.
    catalogLock_.readLock().lock();
    try {
      for (String dbName : getDbNames(null)) {
        Db db = getDb(dbName);
        if (db == null) {
          LOG.error(
              "Database: "
                  + dbName
                  + " was expected to be in the catalog "
                  + "cache. Skipping database and all child objects for this update.");
          continue;
        }
        if (db.isSystemDb()) continue;
        TCatalogObject catalogDb =
            new TCatalogObject(TCatalogObjectType.DATABASE, db.getCatalogVersion());
        catalogDb.setDb(db.toThrift());
        resp.addToObjects(catalogDb);

        for (String tblName : db.getAllTableNames()) {
          TCatalogObject catalogTbl =
              new TCatalogObject(TCatalogObjectType.TABLE, Catalog.INITIAL_CATALOG_VERSION);

          Table tbl = db.getTable(tblName);
          if (tbl == null) {
            LOG.error(
                "Table: "
                    + tblName
                    + " was expected to be in the catalog "
                    + "cache. Skipping table for this update.");
            continue;
          }

          // Only add the extended metadata if this table's version is >=
          // the fromVersion.
          if (tbl.getCatalogVersion() >= fromVersion) {
            try {
              catalogTbl.setTable(tbl.toThrift());
            } catch (Exception e) {
              LOG.debug(
                  String.format(
                      "Error calling toThrift() on table %s.%s: %s",
                      dbName, tblName, e.getMessage()),
                  e);
              continue;
            }
            catalogTbl.setCatalog_version(tbl.getCatalogVersion());
          } else {
            catalogTbl.setTable(new TTable(dbName, tblName));
          }
          resp.addToObjects(catalogTbl);
        }

        for (Function fn : db.getFunctions(null, new PatternMatcher())) {
          TCatalogObject function =
              new TCatalogObject(TCatalogObjectType.FUNCTION, fn.getCatalogVersion());
          function.setFn(fn.toThrift());
          resp.addToObjects(function);
        }
      }

      for (DataSource dataSource : getDataSources()) {
        TCatalogObject catalogObj =
            new TCatalogObject(TCatalogObjectType.DATA_SOURCE, dataSource.getCatalogVersion());
        catalogObj.setData_source(dataSource.toThrift());
        resp.addToObjects(catalogObj);
      }
      for (HdfsCachePool cachePool : hdfsCachePools_) {
        TCatalogObject pool =
            new TCatalogObject(TCatalogObjectType.HDFS_CACHE_POOL, cachePool.getCatalogVersion());
        pool.setCache_pool(cachePool.toThrift());
        resp.addToObjects(pool);
      }

      // Get all roles
      for (Role role : authPolicy_.getAllRoles()) {
        TCatalogObject thriftRole = new TCatalogObject();
        thriftRole.setRole(role.toThrift());
        thriftRole.setCatalog_version(role.getCatalogVersion());
        thriftRole.setType(role.getCatalogObjectType());
        resp.addToObjects(thriftRole);

        for (RolePrivilege p : role.getPrivileges()) {
          TCatalogObject privilege = new TCatalogObject();
          privilege.setPrivilege(p.toThrift());
          privilege.setCatalog_version(p.getCatalogVersion());
          privilege.setType(p.getCatalogObjectType());
          resp.addToObjects(privilege);
        }
      }

      // Each update should contain a single "TCatalog" object which is used to
      // pass overall state on the catalog, such as the current version and the
      // catalog service id.
      TCatalogObject catalog = new TCatalogObject();
      catalog.setType(TCatalogObjectType.CATALOG);
      // By setting the catalog version to the latest catalog version at this point,
      // it ensure impalads will always bump their versions, even in the case where
      // an object has been dropped.
      catalog.setCatalog_version(getCatalogVersion());
      catalog.setCatalog(new TCatalog(catalogServiceId_));
      resp.addToObjects(catalog);

      // The max version is the max catalog version of all items in the update.
      resp.setMax_catalog_version(getCatalogVersion());
      return resp;
    } finally {
      catalogLock_.readLock().unlock();
    }
  }