/** Validates the properties of the chosen cache pool. Throws on error. */ public static void validateCachePool( THdfsCachingOp op, Long directiveId, TableName table, HdfsPartition partition) throws ImpalaRuntimeException { CacheDirectiveEntry entry = getDirective(directiveId); Preconditions.checkNotNull(entry); if (!op.getCache_pool_name().equals(entry.getInfo().getPool())) { throw new ImpalaRuntimeException( String.format( "Cannot cache partition in " + "pool '%s' because it is already cached in '%s'. To change the cache " + "pool for this partition, first uncache using: ALTER TABLE %s.%s " + "%sSET UNCACHED", op.getCache_pool_name(), entry.getInfo().getPool(), table.getDb(), table, // Insert partition string if partition non null partition != null ? String.format( " PARTITION(%s) ", partition.getPartitionName().replaceAll("/", ", ")) : "")); } }
/** * Returns a boolean indicating if the given thrift caching operation would perform an update on * an already existing cache directive. */ public static boolean isUpdateOp(THdfsCachingOp op, Map<String, String> params) throws ImpalaRuntimeException { Long directiveId = Long.parseLong(params.get(CACHE_DIR_ID_PROP_NAME)); CacheDirectiveEntry entry = getDirective(directiveId); Preconditions.checkNotNull(entry); // Verify cache pool if (!op.getCache_pool_name().equals(entry.getInfo().getPool())) { return false; } // Check cache replication factor if ((op.isSetReplication() && op.getReplication() != entry.getInfo().getReplication()) || (!op.isSetReplication() && entry.getInfo().getReplication() != JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR)) { return true; } return false; }
/** * Validates and returns true if a parameter map contains a cache directive ID and validates it * against the NameNode to make sure it exists. If the cache directive ID does not exist, we * remove the value from the parameter map, issue a log message and return false. As the value is * not written back to the Hive MS from this method, the result will be only valid until the next * metadata fetch. Lastly, we update the cache replication factor in the parameters with the value * read from HDFS. */ public static boolean validateCacheParams(Map<String, String> params) { Long directiveId = getCacheDirectiveId(params); if (directiveId == null) return false; CacheDirectiveEntry entry = null; try { entry = getDirective(directiveId); } catch (ImpalaRuntimeException e) { if (e.getCause() != null && e.getCause() instanceof RemoteException) { // This exception signals that the cache directive no longer exists. LOG.error("Cache directive does not exist", e); params.remove(CACHE_DIR_ID_PROP_NAME); params.remove(CACHE_DIR_REPLICATION_PROP_NAME); } else { // This exception signals that there was a connection problem with HDFS. LOG.error("IO Exception, possible connectivity issues with HDFS", e); } return false; } Preconditions.checkNotNull(entry); // On the upgrade path the property might not exist, if it exists // and is different from the one from the meta store, issue a warning. String replicationFactor = params.get(CACHE_DIR_REPLICATION_PROP_NAME); if (replicationFactor != null && Short.parseShort(replicationFactor) != entry.getInfo().getReplication()) { LOG.info( "Replication factor for entry in HDFS differs from value in Hive MS: " + entry.getInfo().getPath().toString() + " " + entry.getInfo().getReplication().toString() + " != " + params.get(CACHE_DIR_REPLICATION_PROP_NAME)); } params.put(CACHE_DIR_REPLICATION_PROP_NAME, String.valueOf(entry.getInfo().getReplication())); return true; }
/** * Given a cache directive ID, returns the replication factor for the directive. Returns null if * no outstanding cache directives match this ID. */ public static Short getCacheReplication(long directiveId) throws ImpalaRuntimeException { CacheDirectiveEntry entry = getDirective(directiveId); return entry != null ? entry.getInfo().getReplication() : null; }
/** * Given a cache directive ID, returns the pool the directive is cached in. Returns null if no * outstanding cache directive match this ID. */ public static String getCachePool(long directiveId) throws ImpalaRuntimeException { CacheDirectiveEntry entry = getDirective(directiveId); return entry == null ? null : entry.getInfo().getPool(); }