/** Generates name for prefix partial partition specification. */ public String getName() throws HiveException { try { return Warehouse.makePartName(fields, values); } catch (MetaException e) { throw new HiveException("Unable to create partial name", e); } }
private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames( Iterable<? extends HivePartitionName> partitionNames) throws Exception { requireNonNull(partitionNames, "partitionNames is null"); checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty"); HivePartitionName firstPartition = Iterables.get(partitionNames, 0); HiveTableName hiveTableName = firstPartition.getHiveTableName(); String databaseName = hiveTableName.getDatabaseName(); String tableName = hiveTableName.getTableName(); List<String> partitionsToFetch = new ArrayList<>(); for (HivePartitionName partitionName : partitionNames) { checkArgument( partitionName.getHiveTableName().equals(hiveTableName), "Expected table name %s but got %s", hiveTableName, partitionName.getHiveTableName()); partitionsToFetch.add(partitionName.getPartitionName()); } List<String> partitionColumnNames = ImmutableList.copyOf( Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet()); try { return retry() .stopOn(NoSuchObjectException.class) .stopOnIllegalExceptions() .run( "getPartitionsByNames", stats .getGetPartitionsByNames() .wrap( () -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions = ImmutableMap.builder(); for (Partition partition : client.getPartitionsByNames( databaseName, tableName, partitionsToFetch)) { String partitionId = FileUtils.makePartName( partitionColumnNames, partition.getValues(), null); partitions.put( HivePartitionName.partition(databaseName, tableName, partitionId), Optional.of(partition)); } return partitions.build(); } })); } catch (NoSuchObjectException e) { // assume none of the partitions in the batch are available return stream(partitionNames.spliterator(), false) .collect(toMap(identity(), (name) -> Optional.empty())); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
/** * Creates path where partitions matching prefix should lie in filesystem * * @param tbl table in which partition is * @return expected location of partitions matching prefix in filesystem */ public Path createPath(Table tbl) throws HiveException { String prefixSubdir; try { prefixSubdir = Warehouse.makePartName(fields, values); } catch (MetaException e) { throw new HiveException("Unable to get partitions directories prefix", e); } Path tableDir = tbl.getDataLocation(); if (tableDir == null) { throw new HiveException("Table has no location set"); } return new Path(tableDir, prefixSubdir); }
/** * Get a prefix of the given parition's string representation. The sencond argument, level, is * used for the prefix length. For example, partition (ds='2010-01-01', hr='00', min='00'), level * 1 will reture 'ds=2010-01-01', and level 2 will return 'ds=2010-01-01/hr=00'. * * @param p partition object * @param level level for prefix depth * @return prefix of partition's string representation * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { List<FieldSchema> ffields = p.getTable().getPartCols(); List<FieldSchema> fields = new ArrayList<FieldSchema>(level); List<String> fvalues = p.getValues(); List<String> values = new ArrayList<String>(level); for (int i = 0; i < level; i++) { FieldSchema fs = ffields.get(i); String s = fvalues.get(i); fields.add(fs); values.add(s); } try { return Warehouse.makePartName(fields, values); } catch (MetaException e) { throw new HiveException("Wasn't able to generate name" + " for partial specification"); } }
/** * For equals and not-equals, we can make the JDO query much faster by filtering based on the * partition name. For a condition like ds="2010-10-01", we can see if there are any partitions * with a name that contains the substring "ds=2010-10-01/" False matches aren't possible since * "=" is escaped for partition names and the trailing '/' ensures that we won't get a match with * ds=2010-10-011 Note that filters on integral type equality also work correctly by virtue of * comparing them as part of ds=1234 string. * * <p>Two cases to keep in mind: Case with only one partition column (no '/'s) Case where the * partition key column is at the end of the name. (no tailing '/') * * @param keyName name of the partition column e.g. ds. * @param value The value to compare to. * @param paramName name of the parameter to use for JDOQL. * @param params a map from the parameter name to their values. * @param keyPos The index of the requisite partition column in the list of such columns. * @param keyCount Partition column count for the table. * @param isEq whether the operator is equals, or not-equals. * @param fltr Filter builder used to append the filter, or report errors. */ private static void makeFilterForEquals( String keyName, String value, String paramName, Map<String, Object> params, int keyPos, int keyCount, boolean isEq, FilterBuilder fltr) throws MetaException { Map<String, String> partKeyToVal = new HashMap<String, String>(); partKeyToVal.put(keyName, value); // If a partition has multiple partition keys, we make the assumption that // makePartName with one key will return a substring of the name made // with both all the keys. String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false); if (keyCount == 1) { // Case where this is no other partition columns params.put(paramName, escapedNameFragment); fltr.append("partitionName ").append(isEq ? "== " : "!= ").append(paramName); } else if (keyPos + 1 == keyCount) { // Case where the partition column is at the end of the name. There will // be a leading '/' but no trailing '/' params.put(paramName, "/" + escapedNameFragment); fltr.append(isEq ? "" : "!").append("partitionName.endsWith(").append(paramName).append(")"); } else if (keyPos == 0) { // Case where the partition column is at the beginning of the name. There will // be a trailing '/' but no leading '/' params.put(paramName, escapedNameFragment + "/"); fltr.append(isEq ? "" : "!") .append("partitionName.startsWith(") .append(paramName) .append(")"); } else { // Case where the partition column is in the middle of the name. There will // be a leading '/' and an trailing '/' params.put(paramName, "/" + escapedNameFragment + "/"); fltr.append("partitionName.indexOf(") .append(paramName) .append(")") .append(isEq ? ">= 0" : "< 0"); } }
/** todo: what should this do on failure? Should it rethrow? Invalidate stats? */ void gatherStats() throws IOException { if (!ci.isMajorCompaction()) { return; } if (columnList.isEmpty()) { LOG.debug( "No existing stats for " + ci.dbname + "." + ci.tableName + " found. Will not run analyze."); return; // nothing to do } // e.g. analyze table page_view partition(dt='10/15/2014',country=’US’) // compute statistics for columns viewtime StringBuilder sb = new StringBuilder("analyze table ").append(ci.dbname).append(".").append(ci.tableName); if (ci.partName != null) { try { sb.append(" partition("); Map<String, String> partitionColumnValues = Warehouse.makeEscSpecFromName(ci.partName); for (Map.Entry<String, String> ent : partitionColumnValues.entrySet()) { sb.append(ent.getKey()).append("='").append(ent.getValue()).append("'"); } sb.append(")"); } catch (MetaException ex) { throw new IOException(ex); } } sb.append(" compute statistics for columns "); for (String colName : columnList) { sb.append(colName).append(","); } sb.setLength(sb.length() - 1); // remove trailing , LOG.info("running '" + sb.toString() + "'"); Driver d = new Driver(conf, userName); SessionState localSession = null; if (SessionState.get() == null) { localSession = SessionState.start(new SessionState(conf)); } try { CommandProcessorResponse cpr = d.run(sb.toString()); if (cpr.getResponseCode() != 0) { throw new IOException( "Could not update stats for table " + ci.getFullTableName() + (ci.partName == null ? "" : "/" + ci.partName) + " due to: " + cpr); } } catch (CommandNeedRetryException cnre) { throw new IOException( "Could not update stats for table " + ci.getFullTableName() + (ci.partName == null ? "" : "/" + ci.partName) + " due to: " + cnre.getMessage()); } finally { if (localSession != null) { localSession.close(); } } }