/** Generates name for prefix partial partition specification. */
public String getName() throws HiveException {
try {
return Warehouse.makePartName(fields, values);
} catch (MetaException e) {
throw new HiveException("Unable to create partial name", e);
}
}
private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(
Iterable<? extends HivePartitionName> partitionNames) throws Exception {
requireNonNull(partitionNames, "partitionNames is null");
checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");
HivePartitionName firstPartition = Iterables.get(partitionNames, 0);
HiveTableName hiveTableName = firstPartition.getHiveTableName();
String databaseName = hiveTableName.getDatabaseName();
String tableName = hiveTableName.getTableName();
List<String> partitionsToFetch = new ArrayList<>();
for (HivePartitionName partitionName : partitionNames) {
checkArgument(
partitionName.getHiveTableName().equals(hiveTableName),
"Expected table name %s but got %s",
hiveTableName,
partitionName.getHiveTableName());
partitionsToFetch.add(partitionName.getPartitionName());
}
List<String> partitionColumnNames =
ImmutableList.copyOf(
Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet());
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getPartitionsByNames",
stats
.getGetPartitionsByNames()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions =
ImmutableMap.builder();
for (Partition partition :
client.getPartitionsByNames(
databaseName, tableName, partitionsToFetch)) {
String partitionId =
FileUtils.makePartName(
partitionColumnNames, partition.getValues(), null);
partitions.put(
HivePartitionName.partition(databaseName, tableName, partitionId),
Optional.of(partition));
}
return partitions.build();
}
}));
} catch (NoSuchObjectException e) {
// assume none of the partitions in the batch are available
return stream(partitionNames.spliterator(), false)
.collect(toMap(identity(), (name) -> Optional.empty()));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
/**
* Creates path where partitions matching prefix should lie in filesystem
*
* @param tbl table in which partition is
* @return expected location of partitions matching prefix in filesystem
*/
public Path createPath(Table tbl) throws HiveException {
String prefixSubdir;
try {
prefixSubdir = Warehouse.makePartName(fields, values);
} catch (MetaException e) {
throw new HiveException("Unable to get partitions directories prefix", e);
}
Path tableDir = tbl.getDataLocation();
if (tableDir == null) {
throw new HiveException("Table has no location set");
}
return new Path(tableDir, prefixSubdir);
}
/**
* Get a prefix of the given parition's string representation. The sencond argument, level, is
* used for the prefix length. For example, partition (ds='2010-01-01', hr='00', min='00'), level
* 1 will reture 'ds=2010-01-01', and level 2 will return 'ds=2010-01-01/hr=00'.
*
* @param p partition object
* @param level level for prefix depth
* @return prefix of partition's string representation
* @throws HiveException
*/
public static String getPartialName(Partition p, int level) throws HiveException {
List<FieldSchema> ffields = p.getTable().getPartCols();
List<FieldSchema> fields = new ArrayList<FieldSchema>(level);
List<String> fvalues = p.getValues();
List<String> values = new ArrayList<String>(level);
for (int i = 0; i < level; i++) {
FieldSchema fs = ffields.get(i);
String s = fvalues.get(i);
fields.add(fs);
values.add(s);
}
try {
return Warehouse.makePartName(fields, values);
} catch (MetaException e) {
throw new HiveException("Wasn't able to generate name" + " for partial specification");
}
}
/**
* For equals and not-equals, we can make the JDO query much faster by filtering based on the
* partition name. For a condition like ds="2010-10-01", we can see if there are any partitions
* with a name that contains the substring "ds=2010-10-01/" False matches aren't possible since
* "=" is escaped for partition names and the trailing '/' ensures that we won't get a match with
* ds=2010-10-011 Note that filters on integral type equality also work correctly by virtue of
* comparing them as part of ds=1234 string.
*
* <p>Two cases to keep in mind: Case with only one partition column (no '/'s) Case where the
* partition key column is at the end of the name. (no tailing '/')
*
* @param keyName name of the partition column e.g. ds.
* @param value The value to compare to.
* @param paramName name of the parameter to use for JDOQL.
* @param params a map from the parameter name to their values.
* @param keyPos The index of the requisite partition column in the list of such columns.
* @param keyCount Partition column count for the table.
* @param isEq whether the operator is equals, or not-equals.
* @param fltr Filter builder used to append the filter, or report errors.
*/
private static void makeFilterForEquals(
String keyName,
String value,
String paramName,
Map<String, Object> params,
int keyPos,
int keyCount,
boolean isEq,
FilterBuilder fltr)
throws MetaException {
Map<String, String> partKeyToVal = new HashMap<String, String>();
partKeyToVal.put(keyName, value);
// If a partition has multiple partition keys, we make the assumption that
// makePartName with one key will return a substring of the name made
// with both all the keys.
String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false);
if (keyCount == 1) {
// Case where this is no other partition columns
params.put(paramName, escapedNameFragment);
fltr.append("partitionName ").append(isEq ? "== " : "!= ").append(paramName);
} else if (keyPos + 1 == keyCount) {
// Case where the partition column is at the end of the name. There will
// be a leading '/' but no trailing '/'
params.put(paramName, "/" + escapedNameFragment);
fltr.append(isEq ? "" : "!").append("partitionName.endsWith(").append(paramName).append(")");
} else if (keyPos == 0) {
// Case where the partition column is at the beginning of the name. There will
// be a trailing '/' but no leading '/'
params.put(paramName, escapedNameFragment + "/");
fltr.append(isEq ? "" : "!")
.append("partitionName.startsWith(")
.append(paramName)
.append(")");
} else {
// Case where the partition column is in the middle of the name. There will
// be a leading '/' and an trailing '/'
params.put(paramName, "/" + escapedNameFragment + "/");
fltr.append("partitionName.indexOf(")
.append(paramName)
.append(")")
.append(isEq ? ">= 0" : "< 0");
}
}
/** todo: what should this do on failure? Should it rethrow? Invalidate stats? */
void gatherStats() throws IOException {
if (!ci.isMajorCompaction()) {
return;
}
if (columnList.isEmpty()) {
LOG.debug(
"No existing stats for "
+ ci.dbname
+ "."
+ ci.tableName
+ " found. Will not run analyze.");
return; // nothing to do
}
// e.g. analyze table page_view partition(dt='10/15/2014',country=’US’)
// compute statistics for columns viewtime
StringBuilder sb =
new StringBuilder("analyze table ").append(ci.dbname).append(".").append(ci.tableName);
if (ci.partName != null) {
try {
sb.append(" partition(");
Map<String, String> partitionColumnValues = Warehouse.makeEscSpecFromName(ci.partName);
for (Map.Entry<String, String> ent : partitionColumnValues.entrySet()) {
sb.append(ent.getKey()).append("='").append(ent.getValue()).append("'");
}
sb.append(")");
} catch (MetaException ex) {
throw new IOException(ex);
}
}
sb.append(" compute statistics for columns ");
for (String colName : columnList) {
sb.append(colName).append(",");
}
sb.setLength(sb.length() - 1); // remove trailing ,
LOG.info("running '" + sb.toString() + "'");
Driver d = new Driver(conf, userName);
SessionState localSession = null;
if (SessionState.get() == null) {
localSession = SessionState.start(new SessionState(conf));
}
try {
CommandProcessorResponse cpr = d.run(sb.toString());
if (cpr.getResponseCode() != 0) {
throw new IOException(
"Could not update stats for table "
+ ci.getFullTableName()
+ (ci.partName == null ? "" : "/" + ci.partName)
+ " due to: "
+ cpr);
}
} catch (CommandNeedRetryException cnre) {
throw new IOException(
"Could not update stats for table "
+ ci.getFullTableName()
+ (ci.partName == null ? "" : "/" + ci.partName)
+ " due to: "
+ cnre.getMessage());
} finally {
if (localSession != null) {
localSession.close();
}
}
}