Пример #1
0
 /**
  * Verify that all THdfsPartitions included in the descriptor table are referenced by at least one
  * scan range or part of an inserted table. PrintScanRangeLocations will implicitly verify the
  * converse (it'll fail if a scan range references a table/partition descriptor that is not
  * present).
  */
 private void testHdfsPartitionsReferenced(
     TQueryExecRequest execRequest, String query, StringBuilder errorLog) {
   long insertTableId = -1;
   // Collect all partitions that are referenced by a scan range.
   Set<THdfsPartition> scanRangePartitions = Sets.newHashSet();
   if (execRequest.per_node_scan_ranges != null) {
     for (Map.Entry<Integer, List<TScanRangeLocations>> entry :
         execRequest.per_node_scan_ranges.entrySet()) {
       if (entry.getValue() == null) {
         continue;
       }
       for (TScanRangeLocations locations : entry.getValue()) {
         if (locations.scan_range.isSetHdfs_file_split()) {
           THdfsFileSplit split = locations.scan_range.getHdfs_file_split();
           THdfsPartition partition = findPartition(entry.getKey(), split);
           scanRangePartitions.add(partition);
         }
       }
     }
   }
   if (execRequest.isSetFinalize_params()) {
     insertTableId = execRequest.getFinalize_params().getTable_id();
   }
   boolean first = true;
   // Iterate through all partitions of the descriptor table and verify all partitions
   // are referenced.
   if (execRequest.isSetDesc_tbl() && execRequest.desc_tbl.isSetTableDescriptors()) {
     for (TTableDescriptor tableDesc : execRequest.desc_tbl.tableDescriptors) {
       // All partitions of insertTableId are okay.
       if (tableDesc.getId() == insertTableId) continue;
       if (!tableDesc.isSetHdfsTable()) continue;
       THdfsTable hdfsTable = tableDesc.getHdfsTable();
       for (Map.Entry<Long, THdfsPartition> e : hdfsTable.getPartitions().entrySet()) {
         THdfsPartition partition = e.getValue();
         if (!scanRangePartitions.contains(partition)) {
           if (first) errorLog.append("query:\n" + query + "\n");
           errorLog.append(
               " unreferenced partition: HdfsTable: "
                   + tableDesc.getId()
                   + " HdfsPartition: "
                   + partition.getId()
                   + "\n");
           first = false;
         }
       }
     }
   }
 }
Пример #2
0
  private THdfsTable getHdfsTable() {
    Map<Long, THdfsPartition> idToPartition = Maps.newHashMap();
    for (HdfsPartition partition : partitions_) {
      idToPartition.put(partition.getId(), partition.toThrift(true));
    }

    List<String> colNames = new ArrayList<String>();
    for (int i = 0; i < colsByPos_.size(); ++i) {
      colNames.add(colsByPos_.get(i).getName());
    }
    THdfsTable hdfsTable =
        new THdfsTable(
            hdfsBaseDir_, colNames, nullPartitionKeyValue_, nullColumnValue_, idToPartition);
    hdfsTable.setAvroSchema(avroSchema_);
    return hdfsTable;
  }
Пример #3
0
 /**
  * Look up the partition corresponding to the plan node (identified by nodeId) and a file split.
  */
 private THdfsPartition findPartition(int nodeId, THdfsFileSplit split) {
   TPlanNode node = planMap_.get(nodeId);
   Preconditions.checkNotNull(node);
   Preconditions.checkState(node.node_id == nodeId && node.isSetHdfs_scan_node());
   THdfsScanNode scanNode = node.getHdfs_scan_node();
   int tupleId = scanNode.getTuple_id();
   TTupleDescriptor tupleDesc = tupleMap_.get(tupleId);
   Preconditions.checkNotNull(tupleDesc);
   Preconditions.checkState(tupleDesc.id == tupleId);
   TTableDescriptor tableDesc = tableMap_.get(tupleDesc.tableId);
   Preconditions.checkNotNull(tableDesc);
   Preconditions.checkState(tableDesc.id == tupleDesc.tableId && tableDesc.isSetHdfsTable());
   THdfsTable hdfsTable = tableDesc.getHdfsTable();
   THdfsPartition partition = hdfsTable.getPartitions().get(split.partition_id);
   Preconditions.checkNotNull(partition);
   Preconditions.checkState(partition.id == split.partition_id);
   return partition;
 }
Пример #4
0
  @Override
  protected void loadFromThrift(TTable thriftTable) throws TableLoadingException {
    super.loadFromThrift(thriftTable);
    THdfsTable hdfsTable = thriftTable.getHdfs_table();
    hdfsBaseDir_ = hdfsTable.getHdfsBaseDir();
    nullColumnValue_ = hdfsTable.nullColumnValue;
    nullPartitionKeyValue_ = hdfsTable.nullPartitionKeyValue;

    numHdfsFiles_ = 0;
    totalHdfsBytes_ = 0;
    for (Map.Entry<Long, THdfsPartition> part : hdfsTable.getPartitions().entrySet()) {
      HdfsPartition hdfsPart = HdfsPartition.fromThrift(this, part.getKey(), part.getValue());
      numHdfsFiles_ += hdfsPart.getFileDescriptors().size();
      totalHdfsBytes_ += hdfsPart.getSize();
      partitions_.add(hdfsPart);
    }
    uniqueHostPortsCount_ = countUniqueDataNetworkLocations(partitions_);
    avroSchema_ = hdfsTable.isSetAvroSchema() ? hdfsTable.getAvroSchema() : null;
  }