Ejemplo n.º 1
0
  public boolean next(Row.Key key, Writable value) throws IOException {
    RowArray rows = scanner.nextRow();
    if (rows != null) {
      readRowCount++;
      Row.Key rowKey = rows.getRowKey();

      key.set(rowKey.getBytes());

      ((RowArray) value).setRowKey(rowKey);
      ((RowArray) value).setRows(rows.getRows());
      end = false;
      return true;
    } else {
      end = true;
      return false;
    }
  }
Ejemplo n.º 2
0
  public TableJoinRecordReader(
      JobConf jobConf, CloudataConf conf, TableSplit tableSplit, Reporter reporter)
      throws IOException {
    this.conf = conf;

    String mergeEvaluatorClass = tableSplit.getInputTableInfo().getMergeEvaluatorClass();
    MergeEvaluator mergeEvaluator = null;
    if (mergeEvaluatorClass != null && mergeEvaluatorClass.length() > 0) {
      try {
        mergeEvaluator = (MergeEvaluator) Class.forName(mergeEvaluatorClass).newInstance();
      } catch (Exception e) {
        LOG.error("mergeEvaluator:" + mergeEvaluatorClass + "," + e.getMessage());
        IOException err = new IOException(e.getMessage() + ":" + mergeEvaluatorClass);
        err.initCause(e);
        throw err;
      }
    }

    RowFilter splitRowFilter = tableSplit.getRowFilter();
    InputTableInfo inputTableInfo = tableSplit.getInputTableInfo();

    this.startRowKey = splitRowFilter.getStartRowKey();
    this.endRowKey = splitRowFilter.getEndRowKey();

    RowFilter rowFilter = inputTableInfo.getRowFilter();
    rowFilter.setStartRowKey(startRowKey);
    rowFilter.setEndRowKey(endRowKey);

    CTable ctable = CTable.openTable(conf, inputTableInfo.getTableName());

    TableScanner pivotScanner = null;
    TableScanner targetScanner = null;
    try {
      pivotScanner =
          ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
      Row.Key firstRowKey = null;
      try {
        // 첫번째 Tablet이 아닌 경우에는 첫번째 row는 무시한다.
        if (!startRowKey.equals(Row.Key.MIN_KEY)) {
          Row pivotRow = pivotScanner.nextRow();
          if (pivotRow == null) {
            end = true;
            return;
          }

          if (firstRowKey == null) {
            firstRowKey = pivotRow.getKey();
          }

          if (firstRowKey.equals(pivotRow.getKey())) {
            pivotRow = pivotScanner.nextRow();
            if (pivotRow == null) {
              end = true;
              return;
            }
          }
          pivotScanner.close();
          rowFilter.setStartRowKey(firstRowKey);
          pivotScanner =
              ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
        } else {
          firstRowKey = startRowKey;
        }
      } catch (Exception e) {
        if (pivotScanner != null) {
          pivotScanner.close();
        }
        throw e;
      }

      RowFilter joinRowFilter = inputTableInfo.getJoinRowFilter();

      if (mergeEvaluator != null) {
        if (!firstRowKey.equals(Row.Key.MIN_KEY)) {
          joinRowFilter.setStartRowKey(mergeEvaluator.parseTargetRowKey(firstRowKey, 0));
        } else {
          joinRowFilter.setStartRowKey(Row.Key.MIN_KEY);
        }
        if (!rowFilter.getEndRowKey().equals(Row.Key.MAX_KEY)) {
          joinRowFilter.setEndRowKey(mergeEvaluator.parseTargetRowKey(rowFilter.getEndRowKey(), 0));
        } else {
          joinRowFilter.setEndRowKey(Row.Key.MAX_KEY);
        }
      } else {
        joinRowFilter.setStartRowKey(firstRowKey);
        joinRowFilter.setEndRowKey(rowFilter.getEndRowKey());
      }

      reporter.setStatus(
          inputTableInfo.getTableName()
              + ":"
              + startRowKey
              + " ~ "
              + endRowKey
              + ", "
              + inputTableInfo.getJoinTableName()
              + ":"
              + joinRowFilter.getStartRowKey()
              + " ~ "
              + joinRowFilter.getEndRowKey());

      // pivot table의 startRow, endRow에 대응하는 target table의 scanner 생성
      CTable targetTable = CTable.openTable(conf, inputTableInfo.getJoinTableName());
      targetScanner =
          ScannerFactory.openScanner(targetTable, joinRowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);

      this.scanner =
          new MergeScanner(new TableScanner[] {pivotScanner, targetScanner}, mergeEvaluator);
    } catch (Exception e) {
      if (targetScanner != null) {
        targetScanner.close();
      }
      IOException err = new IOException(e.getMessage());
      err.initCause(e);
      throw err;
    }
  }