/** * Method to fetch table data * * @param table table name * @param database database * @return list of columns in comma seperated way * @throws Exception if any error occurs */ private List<String> getTableData(String table, String database) throws Exception { HiveConf conf = new HiveConf(); conf.addResource("hive-site.xml"); ArrayList<String> results = new ArrayList<String>(); ArrayList<String> temp = new ArrayList<String>(); Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; if (!tbl.getPartCols().isEmpty()) { List<Partition> partitions = hive.getPartitions(tbl); List<PartitionDesc> partDesc = new ArrayList<PartitionDesc>(); List<String> partLocs = new ArrayList<String>(); for (Partition part : partitions) { partLocs.add(part.getLocation()); partDesc.add(Utilities.getPartitionDesc(part)); } work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl)); work.setLimit(100); } else { work = new FetchWork(tbl.getDataLocation().toString(), Utilities.getTableDesc(tbl)); } FetchTask task = new FetchTask(); task.setWork(work); task.initialize(conf, null, null); task.fetch(temp); for (String str : temp) { results.add(str.replace("\t", ",")); } return results; }
public static void createHiveTable(String db, String table, List<FieldSchema> columns) throws Exception { Table tbl1 = new Table(db, table); tbl1.setFields(columns); Hive.get().createTable(tbl1); Assert.assertNotNull(Hive.get().getTable(db, table)); log.info("Created table : " + table); }
/** Clear out any side effects of running tests */ public void clearTablesCreatedDuringTests() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) != null) { return; } // Delete any tables other than the source tables // and any databases other than the default database. for (String dbName : db.getAllDatabases()) { SessionState.get().setCurrentDatabase(dbName); for (String tblName : db.getAllTables()) { if (!DEFAULT_DATABASE_NAME.equals(dbName)) { Table tblObj = db.getTable(tblName); // dropping index table can not be dropped directly. Dropping the base // table will automatically drop all its index table if (tblObj.isIndexTable()) { continue; } db.dropTable(dbName, tblName); } else { // this table is defined in srcTables, drop all indexes on it List<Index> indexes = db.getIndexes(dbName, tblName, (short) -1); if (indexes != null && indexes.size() > 0) { for (Index index : indexes) { db.dropIndex(dbName, tblName, index.getIndexName(), true, true); } } } } if (!DEFAULT_DATABASE_NAME.equals(dbName)) { // Drop cascade, may need to drop functions db.dropDatabase(dbName, true, true, true); } } // delete remaining directories for external tables (can affect stats for following tests) try { Path p = new Path(testWarehouse); FileSystem fileSystem = p.getFileSystem(conf); if (fileSystem.exists(p)) { for (FileStatus status : fileSystem.listStatus(p)) { if (status.isDir()) { fileSystem.delete(status.getPath(), true); } } } } catch (IllegalArgumentException e) { // ignore.. provides invalid url sometimes intentionally } SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME); List<String> roleNames = db.getAllRoleNames(); for (String roleName : roleNames) { if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) { db.dropRole(roleName); } } }
/** * set current session to existing session object if a thread is running multiple sessions - it * must call this method with the new session object when switching from one session to another. */ public static SessionState start(SessionState startSs) { setCurrentSessionState(startSs); if (startSs.hiveHist == null) { if (startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) { startSs.hiveHist = new HiveHistoryImpl(startSs); } else { // Hive history is disabled, create a no-op proxy startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy(); } } // Get the following out of the way when you start the session these take a // while and should be done when we start up. try { // Hive object instance should be created with a copy of the conf object. If the conf is // shared with SessionState, other parts of the code might update the config, but // Hive.get(HiveConf) would not recognize the case when it needs refreshing Hive.get(new HiveConf(startSs.conf)).getMSC(); UserGroupInformation sessionUGI = Utils.getUGI(); FileSystem.get(startSs.conf); // Create scratch dirs for this session startSs.createSessionDirs(sessionUGI.getShortUserName()); // Set temp file containing results to be sent to HiveClient if (startSs.getTmpOutputFile() == null) { try { startSs.setTmpOutputFile(createTempFile(startSs.getConf())); } catch (IOException e) { throw new RuntimeException(e); } } } catch (Exception e) { // Catch-all due to some exec time dependencies on session state // that would cause ClassNoFoundException otherwise throw new RuntimeException(e); } if (HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") && (startSs.isHiveServerQuery == false)) { try { if (startSs.tezSessionState == null) { startSs.tezSessionState = new TezSessionState(startSs.getSessionId()); } if (!startSs.tezSessionState.isOpen()) { startSs.tezSessionState.open(startSs.conf); // should use conf on session start-up } } catch (Exception e) { throw new RuntimeException(e); } } else { LOG.info("No Tez session required at this point. hive.execution.engine=mr."); } return startSs; }
// obtain delegation token for the give user from metastore public synchronized String getDelegationTokenFromMetaStore(String owner) throws HiveSQLException, UnsupportedOperationException, LoginException, IOException { if (!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL) || !hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { throw new UnsupportedOperationException( "delegation token is can only be obtained for a secure remote metastore"); } try { Hive.closeCurrent(); return Hive.get(hiveConf).getDelegationToken(owner, owner); } catch (HiveException e) { if (e.getCause() instanceof UnsupportedOperationException) { throw (UnsupportedOperationException) e.getCause(); } else { throw new HiveSQLException("Error connect metastore to setup impersonation", e); } } }
public void init() throws Exception { testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); SessionState.start(conf); conf.set("hive.execution.engine", execEngine); db = Hive.get(conf); pd = new ParseDriver(); sem = new SemanticAnalyzer(conf); }
/** * This code block iterates over indexes on the table and populates the indexToKeys map for all * the indexes that satisfy the rewrite criteria. * * @param indexTables * @return * @throws SemanticException */ Map<Index, Set<String>> getIndexToKeysMap(List<Index> indexTables) throws SemanticException { Index index = null; Hive hiveInstance = hiveDb; Map<Index, Set<String>> indexToKeysMap = new LinkedHashMap<Index, Set<String>>(); for (int idxCtr = 0; idxCtr < indexTables.size(); idxCtr++) { final Set<String> indexKeyNames = new LinkedHashSet<String>(); index = indexTables.get(idxCtr); // Getting index key columns StorageDescriptor sd = index.getSd(); List<FieldSchema> idxColList = sd.getCols(); for (FieldSchema fieldSchema : idxColList) { indexKeyNames.add(fieldSchema.getName()); } assert indexKeyNames.size() == 1; // Check that the index schema is as expected. This code block should // catch problems of this rewrite breaking when the AggregateIndexHandler // index is changed. List<String> idxTblColNames = new ArrayList<String>(); try { Table idxTbl = hiveInstance.getTable(index.getDbName(), index.getIndexTableName()); for (FieldSchema idxTblCol : idxTbl.getCols()) { idxTblColNames.add(idxTblCol.getName()); } } catch (HiveException e) { LOG.error( "Got exception while locating index table, " + "skipping " + getName() + " optimization"); LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); throw new SemanticException(e.getMessage(), e); } assert (idxTblColNames.contains(IDX_BUCKET_COL)); assert (idxTblColNames.contains(IDX_OFFSETS_ARRAY_COL)); // we add all index tables which can be used for rewrite // and defer the decision of using a particular index for later // this is to allow choosing a index if a better mechanism is // designed later to chose a better rewrite indexToKeysMap.put(index, indexKeyNames); } return indexToKeysMap; }
@Before public void setup() throws Exception { conf = new HiveConf(); conf.setVar( ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY, SentryHiveAuthorizationTaskFactoryImpl.class.getName()); db = Mockito.mock(Hive.class); table = new Table(DB, TABLE); partition = new Partition(table); context = new Context(conf); parseDriver = new ParseDriver(); analyzer = new DDLSemanticAnalyzer(conf, db); SessionState.start(conf); Mockito.when(db.getTable(TABLE, false)).thenReturn(table); Mockito.when(db.getPartition(table, new HashMap<String, String>(), false)) .thenReturn(partition); HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator(); auth.setConf(conf); currentUser = auth.getUserName(); }
/** * Determines if one can insert into partition(s), or there's a conflict with archive. It can be * because partition is itself archived or it is to be created inside existing archive. The second * case is when partition doesn't exist yet, but it would be inside of an archive if it existed. * This one is quite tricky to check, we need to find at least one partition inside of the parent * directory. If it is archived and archiving level tells that the archival was done of directory * partition is in it means we cannot insert; otherwise we can. This method works both for full * specifications and partial ones - in second case it checks if any partition that could possibly * match such specification is inside archive. * * @param db - Hive object * @param tbl - table where partition is * @param partSpec - partition specification with possible nulls in case of dynamic partiton * inserts * @return null if partition can be inserted, string with colliding archive name when it can't * @throws HiveException */ public static String conflictingArchiveNameOrNull( Hive db, Table tbl, LinkedHashMap<String, String> partSpec) throws HiveException { List<FieldSchema> partKeys = tbl.getPartitionKeys(); int partSpecLevel = 0; for (FieldSchema partKey : partKeys) { if (!partSpec.containsKey(partKey.getName())) { break; } partSpecLevel++; } if (partSpecLevel != partSpec.size()) { throw new HiveException("partspec " + partSpec + " is wrong for table " + tbl.getTableName()); } Map<String, String> spec = new HashMap<String, String>(partSpec); List<String> reversedKeys = new LinkedList<String>(); for (FieldSchema fs : tbl.getPartCols()) { if (spec.containsKey(fs.getName())) { reversedKeys.add(0, fs.getName()); } } for (String rk : reversedKeys) { List<Partition> parts = db.getPartitions(tbl, spec, (short) 1); if (parts.size() != 0) { Partition p = parts.get(0); if (!isArchived(p)) { // if archiving was done at this or at upper level, every matched // partition would be archived, so it not being archived means // no archiving was done neither at this nor at upper level return null; } else if (getArchivingLevel(p) > spec.size()) { // if archiving was done at this or at upper level its level // would be lesser or equal to specification size // it is not, which means no archiving at this or upper level return null; } else { return getPartialName(p, getArchivingLevel(p)); } } spec.remove(rk); } return null; }
public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { this.queryPlan = queryPlan; isdone = false; started = false; setInitialized(); this.conf = conf; try { db = Hive.get(conf); } catch (HiveException e) { // Bail out ungracefully - we should never hit // this here - but would have hit it in SemanticAnalyzer LOG.error(StringUtils.stringifyException(e)); throw new RuntimeException(e); } this.driverContext = driverContext; console = new LogHelper(LOG); }
@BeforeTest @Override public void beforeTest() throws Exception { super.beforeTest(); List<FieldSchema> factColumns = new ArrayList<>(); factColumns.add(new FieldSchema("continent", "int", "")); factColumns.add(new FieldSchema("country", "int", "")); factColumns.add(new FieldSchema("region", "int", "")); factColumns.add(new FieldSchema("city", "int", "")); factColumns.add(new FieldSchema("count", "double", "")); factColumns.add(new FieldSchema("added", "double", "")); try { HiveConf hiveConf = new HiveConf(); SessionState.start(hiveConf); Hive.get().dropTable("default.wikipedia"); createHiveTable("default", "wikipedia", factColumns); Table tbl = CubeMetastoreClient.getInstance(hiveConf).getHiveTable("wikipedia"); tbl.setProperty("druid.table.time.dimension", "time"); } catch (HiveException e) { log.error("Exception while creating hive table", e); } }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { parseContext = pctx; hiveConf = parseContext.getConf(); try { hiveDb = Hive.get(hiveConf); } catch (HiveException e) { LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); throw new SemanticException(e.getMessage(), e); } // Don't try to index optimize the query to build the index HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false); /* Check if the input query passes all the tests to be eligible for a rewrite * If yes, rewrite original query; else, return the current parseContext */ if (shouldApplyOptimization()) { LOG.info("Rewriting Original Query using " + getName() + " optimization."); rewriteOriginalQuery(); } return parseContext; }
/** * This is called outside of the lock. Some of the methods that are called transitively by this * (e.g. addFunction) will take the lock again and then release it, which is ok. */ private FunctionInfo getFunctionInfoFromMetastoreNoLock(String functionName, HiveConf conf) { try { String[] parts = FunctionUtils.getQualifiedFunctionNameParts(functionName); Function func = Hive.get(conf).getFunction(parts[0].toLowerCase(), parts[1]); if (func == null) { return null; } // Found UDF in metastore - now add it to the function registry. FunctionInfo fi = registerPermanentFunction( functionName, func.getClassName(), true, FunctionTask.toFunctionResource(func.getResourceUris())); if (fi == null) { LOG.error(func.getClassName() + " is not a valid UDF class and was not registered"); return null; } return fi; } catch (Throwable e) { LOG.info("Unable to look up " + functionName + " in metastore", e); } return null; }
public static void main(String[] args) throws Exception { OptionsProcessor oproc = new OptionsProcessor(); if (!oproc.process_stage1(args)) { System.exit(1); } SessionState.initHiveLog4j(); CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); ss.in = System.in; try { ss.out = new PrintStream(System.out, true, "UTF-8"); ss.err = new PrintStream(System.err, true, "UTF-8"); } catch (UnsupportedEncodingException e) { System.exit(3); } if (!oproc.process_stage2(ss)) { System.exit(2); } HiveConf conf = ss.getConf(); for (Map.Entry<Object, Object> item : ss.cmdProperties.entrySet()) { conf.set((String) item.getKey(), (String) item.getValue()); } if (!ShimLoader.getHadoopShims().usesJobShell()) { ClassLoader loader = conf.getClassLoader(); String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); if (StringUtils.isNotBlank(auxJars)) { loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ",")); } conf.setClassLoader(loader); Thread.currentThread().setContextClassLoader(loader); } SessionState.start(ss); CliDriver cli = new CliDriver(); Hive hive = Hive.get(); String username = ss.getUserName(); String passwd = ss.passwd; if (!hive.isAUser(username, passwd)) { System.out.println("User or passwd is wrong!"); System.exit(0); } else { System.out.println("Connect to TDW successfully!"); } if (ss.getDbName() == null) { ss.setDbName(MetaStoreUtils.DEFAULT_DATABASE_NAME); } if (ss.execString != null) { System.exit(cli.processLine(ss.execString)); } try { if (ss.fileName != null) { System.exit(cli.processReader(new BufferedReader(new FileReader(ss.fileName)))); } } catch (FileNotFoundException e) { System.err.println("Could not open input file for reading. (" + e.getMessage() + ")"); System.exit(3); } ConsoleReader reader = new ConsoleReader(); reader.setBellEnabled(false); List<SimpleCompletor> completors = new LinkedList<SimpleCompletor>(); completors.add( new SimpleCompletor( new String[] {"set", "from", "create", "load", "describe", "quit", "exit"})); reader.addCompletor(new ArgumentCompletor(completors)); String line; final String HISTORYFILE = ".hivehistory"; String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE; reader.setHistory(new History(new File(historyFile))); int ret = 0; String prefix = ""; String curPrompt = prompt; while ((line = reader.readLine(curPrompt + "> ")) != null) { if (!prefix.equals("")) { prefix += '\n'; } if (line.trim().endsWith(";") && !line.trim().endsWith("\\;")) { line = prefix + line; ret = cli.processLine(line); prefix = ""; curPrompt = prompt; } else { prefix = prefix + line; curPrompt = prompt2; continue; } } System.exit(ret); }
@Override protected void authorizeDDLWork(HiveSemanticAnalyzerHookContext cntxt, Hive hive, DDLWork work) throws HiveException { // DB opereations, none of them are enforced by Hive right now. ShowDatabasesDesc showDatabases = work.getShowDatabasesDesc(); if (showDatabases != null) { authorize( HiveOperation.SHOWDATABASES.getInputRequiredPrivileges(), HiveOperation.SHOWDATABASES.getOutputRequiredPrivileges()); } DropDatabaseDesc dropDb = work.getDropDatabaseDesc(); if (dropDb != null) { Database db = cntxt.getHive().getDatabase(dropDb.getDatabaseName()); authorize(db, Privilege.DROP); } DescDatabaseDesc descDb = work.getDescDatabaseDesc(); if (descDb != null) { Database db = cntxt.getHive().getDatabase(descDb.getDatabaseName()); authorize(db, Privilege.SELECT); } SwitchDatabaseDesc switchDb = work.getSwitchDatabaseDesc(); if (switchDb != null) { Database db = cntxt.getHive().getDatabase(switchDb.getDatabaseName()); authorize(db, Privilege.SELECT); } ShowTablesDesc showTables = work.getShowTblsDesc(); if (showTables != null) { String dbName = showTables.getDbName() == null ? SessionState.get().getCurrentDatabase() : showTables.getDbName(); authorize(cntxt.getHive().getDatabase(dbName), Privilege.SELECT); } ShowTableStatusDesc showTableStatus = work.getShowTblStatusDesc(); if (showTableStatus != null) { String dbName = showTableStatus.getDbName() == null ? SessionState.get().getCurrentDatabase() : showTableStatus.getDbName(); authorize(cntxt.getHive().getDatabase(dbName), Privilege.SELECT); } // TODO: add alter database support in HCat // Table operations. DropTableDesc dropTable = work.getDropTblDesc(); if (dropTable != null) { if (dropTable.getPartSpecs() == null) { // drop table is already enforced by Hive. We only check for table level location even if // the // table is partitioned. } else { // this is actually a ALTER TABLE DROP PARITITION statement for (DropTableDesc.PartSpec partSpec : dropTable.getPartSpecs()) { // partitions are not added as write entries in drop partitions in Hive Table table = hive.getTable(SessionState.get().getCurrentDatabase(), dropTable.getTableName()); List<Partition> partitions = null; try { partitions = hive.getPartitionsByFilter(table, partSpec.getPartSpec().getExprString()); } catch (Exception e) { throw new HiveException(e); } for (Partition part : partitions) { authorize(part, Privilege.DROP); } } } } AlterTableDesc alterTable = work.getAlterTblDesc(); if (alterTable != null) { Table table = hive.getTable(SessionState.get().getCurrentDatabase(), alterTable.getOldName(), false); Partition part = null; if (alterTable.getPartSpec() != null) { part = hive.getPartition(table, alterTable.getPartSpec(), false); } String newLocation = alterTable.getNewLocation(); /* Hcat requires ALTER_DATA privileges for ALTER TABLE LOCATION statements * for the old table/partition location and the new location. */ if (alterTable.getOp() == AlterTableDesc.AlterTableTypes.ALTERLOCATION) { if (part != null) { authorize(part, Privilege.ALTER_DATA); // authorize for the old // location, and new location part.setLocation(newLocation); authorize(part, Privilege.ALTER_DATA); } else { authorize(table, Privilege.ALTER_DATA); // authorize for the old // location, and new location table.getTTable().getSd().setLocation(newLocation); authorize(table, Privilege.ALTER_DATA); } } // other alter operations are already supported by Hive } // we should be careful when authorizing table based on just the // table name. If columns have separate authorization domain, it // must be honored DescTableDesc descTable = work.getDescTblDesc(); if (descTable != null) { String tableName = extractTableName(descTable.getTableName()); authorizeTable(cntxt.getHive(), tableName, Privilege.SELECT); } ShowPartitionsDesc showParts = work.getShowPartsDesc(); if (showParts != null) { String tableName = extractTableName(showParts.getTabName()); authorizeTable(cntxt.getHive(), tableName, Privilege.SELECT); } }
@Override public synchronized void stop() { super.stop(); // Release the HMS connection for this service thread Hive.closeCurrent(); }
@Override public void analyzeInternal(ASTNode ast) throws SemanticException { isLocal = false; isOverWrite = false; Tree fromTree = ast.getChild(0); Tree tableTree = ast.getChild(1); if (ast.getChildCount() == 4) { isLocal = true; isOverWrite = true; } if (ast.getChildCount() == 3) { if (ast.getChild(2).getText().toLowerCase().equals("local")) { isLocal = true; } else { isOverWrite = true; } } // initialize load path URI fromURI; try { String fromPath = stripQuotes(fromTree.getText()); fromURI = initializeFromURI(fromPath); } catch (IOException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e.getMessage()), e); } catch (URISyntaxException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e.getMessage()), e); } // initialize destination table/partition tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree); if (ts.tableHandle.isOffline()) { throw new SemanticException( ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(":Table " + ts.tableName)); } if (ts.tableHandle.isView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } if (ts.tableHandle.isNonNative()) { throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg()); } if (ts.tableHandle.isStoredAsSubDirectories()) { throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg()); } URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation() : ts.tableHandle.getDataLocation(); List<FieldSchema> parts = ts.tableHandle.getPartitionKeys(); if ((parts != null && parts.size() > 0) && (ts.partSpec == null || ts.partSpec.size() == 0)) { throw new SemanticException(ErrorMsg.NEED_PARTITION_ERROR.getMsg()); } // make sure the arguments make sense applyConstraints(fromURI, toURI, fromTree, isLocal); Task<? extends Serializable> rTask = null; // create copy work if (isLocal) { // if the local keyword is specified - we will always make a copy. this // might seem redundant in the case // that the hive warehouse is also located in the local file system - but // that's just a test case. String copyURIStr = ctx.getExternalTmpFileURI(toURI); URI copyURI = URI.create(copyURIStr); rTask = TaskFactory.get(new CopyWork(fromURI.toString(), copyURIStr), conf); fromURI = copyURI; } // create final load/move work String loadTmpPath = ctx.getExternalTmpFileURI(toURI); Map<String, String> partSpec = ts.getPartSpec(); if (partSpec == null) { partSpec = new LinkedHashMap<String, String>(); outputs.add(new WriteEntity(ts.tableHandle)); } else { try { Partition part = Hive.get().getPartition(ts.tableHandle, partSpec, false); if (part != null) { if (part.isOffline()) { throw new SemanticException( ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(ts.tableName + ":" + part.getName())); } outputs.add(new WriteEntity(part)); } else { outputs.add(new WriteEntity(ts.tableHandle)); } } catch (HiveException e) { throw new SemanticException(e); } } LoadTableDesc loadTableWork = new LoadTableDesc( fromURI.toString(), loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite); Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(), getOutputs(), loadTableWork, null, true), conf); if (rTask != null) { rTask.addDependentTask(childTask); } else { rTask = childTask; } rootTasks.add(rTask); // The user asked for stats to be collected. // Some stats like number of rows require a scan of the data // However, some other stats, like number of files, do not require a complete scan // Update the stats which do not require a complete scan. Task<? extends Serializable> statTask = null; if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { StatsWork statDesc = new StatsWork(loadTableWork); statDesc.setNoStatsAggregator(true); statDesc.setClearAggregatorStats(true); statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); statTask = TaskFactory.get(statDesc, conf); } // HIVE-3334 has been filed for load file with index auto update if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) { IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf); try { List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater.generateUpdateTasks(); for (Task<? extends Serializable> updateTask : indexUpdateTasks) { // LOAD DATA will either have a copy & move or just a move, // we always want the update to be dependent on the move childTask.addDependentTask(updateTask); if (statTask != null) { updateTask.addDependentTask(statTask); } } } catch (HiveException e) { console.printInfo( "WARNING: could not auto-update stale indexes, indexes are not out of sync"); } } else if (statTask != null) { childTask.addDependentTask(statTask); } }