コード例 #1
0
ファイル: ArchiveUtils.java プロジェクト: sushrutikhar/hive
 /** Generates name for prefix partial partition specification. */
 public String getName() throws HiveException {
   try {
     return Warehouse.makePartName(fields, values);
   } catch (MetaException e) {
     throw new HiveException("Unable to create partial name", e);
   }
 }
コード例 #2
0
  private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(
      Iterable<? extends HivePartitionName> partitionNames) throws Exception {
    requireNonNull(partitionNames, "partitionNames is null");
    checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");

    HivePartitionName firstPartition = Iterables.get(partitionNames, 0);

    HiveTableName hiveTableName = firstPartition.getHiveTableName();
    String databaseName = hiveTableName.getDatabaseName();
    String tableName = hiveTableName.getTableName();

    List<String> partitionsToFetch = new ArrayList<>();
    for (HivePartitionName partitionName : partitionNames) {
      checkArgument(
          partitionName.getHiveTableName().equals(hiveTableName),
          "Expected table name %s but got %s",
          hiveTableName,
          partitionName.getHiveTableName());
      partitionsToFetch.add(partitionName.getPartitionName());
    }

    List<String> partitionColumnNames =
        ImmutableList.copyOf(
            Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet());

    try {
      return retry()
          .stopOn(NoSuchObjectException.class)
          .stopOnIllegalExceptions()
          .run(
              "getPartitionsByNames",
              stats
                  .getGetPartitionsByNames()
                  .wrap(
                      () -> {
                        try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
                          ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions =
                              ImmutableMap.builder();
                          for (Partition partition :
                              client.getPartitionsByNames(
                                  databaseName, tableName, partitionsToFetch)) {
                            String partitionId =
                                FileUtils.makePartName(
                                    partitionColumnNames, partition.getValues(), null);
                            partitions.put(
                                HivePartitionName.partition(databaseName, tableName, partitionId),
                                Optional.of(partition));
                          }
                          return partitions.build();
                        }
                      }));
    } catch (NoSuchObjectException e) {
      // assume none of the partitions in the batch are available
      return stream(partitionNames.spliterator(), false)
          .collect(toMap(identity(), (name) -> Optional.empty()));
    } catch (TException e) {
      throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
  }
コード例 #3
0
ファイル: ArchiveUtils.java プロジェクト: sushrutikhar/hive
 /**
  * Creates path where partitions matching prefix should lie in filesystem
  *
  * @param tbl table in which partition is
  * @return expected location of partitions matching prefix in filesystem
  */
 public Path createPath(Table tbl) throws HiveException {
   String prefixSubdir;
   try {
     prefixSubdir = Warehouse.makePartName(fields, values);
   } catch (MetaException e) {
     throw new HiveException("Unable to get partitions directories prefix", e);
   }
   Path tableDir = tbl.getDataLocation();
   if (tableDir == null) {
     throw new HiveException("Table has no location set");
   }
   return new Path(tableDir, prefixSubdir);
 }
コード例 #4
0
ファイル: ArchiveUtils.java プロジェクト: sushrutikhar/hive
 /**
  * Get a prefix of the given parition's string representation. The sencond argument, level, is
  * used for the prefix length. For example, partition (ds='2010-01-01', hr='00', min='00'), level
  * 1 will reture 'ds=2010-01-01', and level 2 will return 'ds=2010-01-01/hr=00'.
  *
  * @param p partition object
  * @param level level for prefix depth
  * @return prefix of partition's string representation
  * @throws HiveException
  */
 public static String getPartialName(Partition p, int level) throws HiveException {
   List<FieldSchema> ffields = p.getTable().getPartCols();
   List<FieldSchema> fields = new ArrayList<FieldSchema>(level);
   List<String> fvalues = p.getValues();
   List<String> values = new ArrayList<String>(level);
   for (int i = 0; i < level; i++) {
     FieldSchema fs = ffields.get(i);
     String s = fvalues.get(i);
     fields.add(fs);
     values.add(s);
   }
   try {
     return Warehouse.makePartName(fields, values);
   } catch (MetaException e) {
     throw new HiveException("Wasn't able to generate name" + " for partial specification");
   }
 }
コード例 #5
0
ファイル: ExpressionTree.java プロジェクト: Leolh/hive
 /**
  * For equals and not-equals, we can make the JDO query much faster by filtering based on the
  * partition name. For a condition like ds="2010-10-01", we can see if there are any partitions
  * with a name that contains the substring "ds=2010-10-01/" False matches aren't possible since
  * "=" is escaped for partition names and the trailing '/' ensures that we won't get a match with
  * ds=2010-10-011 Note that filters on integral type equality also work correctly by virtue of
  * comparing them as part of ds=1234 string.
  *
  * <p>Two cases to keep in mind: Case with only one partition column (no '/'s) Case where the
  * partition key column is at the end of the name. (no tailing '/')
  *
  * @param keyName name of the partition column e.g. ds.
  * @param value The value to compare to.
  * @param paramName name of the parameter to use for JDOQL.
  * @param params a map from the parameter name to their values.
  * @param keyPos The index of the requisite partition column in the list of such columns.
  * @param keyCount Partition column count for the table.
  * @param isEq whether the operator is equals, or not-equals.
  * @param fltr Filter builder used to append the filter, or report errors.
  */
 private static void makeFilterForEquals(
     String keyName,
     String value,
     String paramName,
     Map<String, Object> params,
     int keyPos,
     int keyCount,
     boolean isEq,
     FilterBuilder fltr)
     throws MetaException {
   Map<String, String> partKeyToVal = new HashMap<String, String>();
   partKeyToVal.put(keyName, value);
   // If a partition has multiple partition keys, we make the assumption that
   // makePartName with one key will return a substring of the name made
   // with both all the keys.
   String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false);
   if (keyCount == 1) {
     // Case where this is no other partition columns
     params.put(paramName, escapedNameFragment);
     fltr.append("partitionName ").append(isEq ? "== " : "!= ").append(paramName);
   } else if (keyPos + 1 == keyCount) {
     // Case where the partition column is at the end of the name. There will
     // be a leading '/' but no trailing '/'
     params.put(paramName, "/" + escapedNameFragment);
     fltr.append(isEq ? "" : "!").append("partitionName.endsWith(").append(paramName).append(")");
   } else if (keyPos == 0) {
     // Case where the partition column is at the beginning of the name. There will
     // be a trailing '/' but no leading '/'
     params.put(paramName, escapedNameFragment + "/");
     fltr.append(isEq ? "" : "!")
         .append("partitionName.startsWith(")
         .append(paramName)
         .append(")");
   } else {
     // Case where the partition column is in the middle of the name. There will
     // be a leading '/' and an trailing '/'
     params.put(paramName, "/" + escapedNameFragment + "/");
     fltr.append("partitionName.indexOf(")
         .append(paramName)
         .append(")")
         .append(isEq ? ">= 0" : "< 0");
   }
 }
コード例 #6
0
ファイル: Worker.java プロジェクト: pmedishetty/hive
 /** todo: what should this do on failure? Should it rethrow? Invalidate stats? */
 void gatherStats() throws IOException {
   if (!ci.isMajorCompaction()) {
     return;
   }
   if (columnList.isEmpty()) {
     LOG.debug(
         "No existing stats for "
             + ci.dbname
             + "."
             + ci.tableName
             + " found.  Will not run analyze.");
     return; // nothing to do
   }
   // e.g. analyze table page_view partition(dt='10/15/2014',country=’US’)
   // compute statistics for columns viewtime
   StringBuilder sb =
       new StringBuilder("analyze table ").append(ci.dbname).append(".").append(ci.tableName);
   if (ci.partName != null) {
     try {
       sb.append(" partition(");
       Map<String, String> partitionColumnValues = Warehouse.makeEscSpecFromName(ci.partName);
       for (Map.Entry<String, String> ent : partitionColumnValues.entrySet()) {
         sb.append(ent.getKey()).append("='").append(ent.getValue()).append("'");
       }
       sb.append(")");
     } catch (MetaException ex) {
       throw new IOException(ex);
     }
   }
   sb.append(" compute statistics for columns ");
   for (String colName : columnList) {
     sb.append(colName).append(",");
   }
   sb.setLength(sb.length() - 1); // remove trailing ,
   LOG.info("running '" + sb.toString() + "'");
   Driver d = new Driver(conf, userName);
   SessionState localSession = null;
   if (SessionState.get() == null) {
     localSession = SessionState.start(new SessionState(conf));
   }
   try {
     CommandProcessorResponse cpr = d.run(sb.toString());
     if (cpr.getResponseCode() != 0) {
       throw new IOException(
           "Could not update stats for table "
               + ci.getFullTableName()
               + (ci.partName == null ? "" : "/" + ci.partName)
               + " due to: "
               + cpr);
     }
   } catch (CommandNeedRetryException cnre) {
     throw new IOException(
         "Could not update stats for table "
             + ci.getFullTableName()
             + (ci.partName == null ? "" : "/" + ci.partName)
             + " due to: "
             + cnre.getMessage());
   } finally {
     if (localSession != null) {
       localSession.close();
     }
   }
 }