Esempio n. 1
0
    public void map(
        WritableComparable key,
        Writable value,
        OutputCollector<WritableComparable, Writable> collector,
        Reporter reporter)
        throws IOException {
      LOG.info("Start Map");
      if (err != null) {
        throw err;
      }

      DecimalFormat df = new DecimalFormat("00000000000000000000");

      collector.collect(new Text(tableName), new Text(""));

      for (long i = 0; i < 50000; i++) {
        long randNum = rand.nextLong();
        Row.Key rowKey = new Row.Key(df.format(randNum));
        Row row = new Row(rowKey);
        row.addCell("Col1", new Cell(Cell.Key.EMPTY_KEY, this.data));
        ctable.put(row);
        if (i % 1000 == 0) {
          reporter.progress();
        }

        if (i % 10000 == 0) {
          LOG.info("uploaded: " + i);
        }
        collector.collect(new Text(df.format(randNum)), new Text(""));
      }

      LOG.info("End Map");
    }
Esempio n. 2
0
    public void map(
        WritableComparable key,
        Writable value,
        OutputCollector<WritableComparable, Writable> collector,
        Reporter reporter)
        throws IOException {
      if (err != null) {
        throw err;
      }

      String valueStr = value.toString().trim();
      if (valueStr.indexOf("\t") > 0) {
        valueStr = valueStr.substring(0, valueStr.indexOf("\t"));
      }

      if (this.ctable == null) {
        this.ctable = CTable.openTable(conf, valueStr);
        if (this.ctable == null) {
          throw new IOException("No table:" + valueStr);
        }
        return;
      }

      Row.Key rowKey = new Row.Key(valueStr);
      Row row = ctable.get(rowKey);
      if (row == null) {
        throw new IOException("No Data:[" + rowKey + "]");
      }
      Cell cell = row.getFirst("Col1");
      if (cell == null) {
        throw new IOException("No Cell Data:[" + rowKey + "]");
      }
      if (count % 10000 == 0) {
        LOG.info("read: " + count + ", last row key[" + rowKey + "]");
      }
      count++;
    }
  public TableJoinRecordReader(
      JobConf jobConf, CloudataConf conf, TableSplit tableSplit, Reporter reporter)
      throws IOException {
    this.conf = conf;

    String mergeEvaluatorClass = tableSplit.getInputTableInfo().getMergeEvaluatorClass();
    MergeEvaluator mergeEvaluator = null;
    if (mergeEvaluatorClass != null && mergeEvaluatorClass.length() > 0) {
      try {
        mergeEvaluator = (MergeEvaluator) Class.forName(mergeEvaluatorClass).newInstance();
      } catch (Exception e) {
        LOG.error("mergeEvaluator:" + mergeEvaluatorClass + "," + e.getMessage());
        IOException err = new IOException(e.getMessage() + ":" + mergeEvaluatorClass);
        err.initCause(e);
        throw err;
      }
    }

    RowFilter splitRowFilter = tableSplit.getRowFilter();
    InputTableInfo inputTableInfo = tableSplit.getInputTableInfo();

    this.startRowKey = splitRowFilter.getStartRowKey();
    this.endRowKey = splitRowFilter.getEndRowKey();

    RowFilter rowFilter = inputTableInfo.getRowFilter();
    rowFilter.setStartRowKey(startRowKey);
    rowFilter.setEndRowKey(endRowKey);

    CTable ctable = CTable.openTable(conf, inputTableInfo.getTableName());

    TableScanner pivotScanner = null;
    TableScanner targetScanner = null;
    try {
      pivotScanner =
          ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
      Row.Key firstRowKey = null;
      try {
        // 첫번째 Tablet이 아닌 경우에는 첫번째 row는 무시한다.
        if (!startRowKey.equals(Row.Key.MIN_KEY)) {
          Row pivotRow = pivotScanner.nextRow();
          if (pivotRow == null) {
            end = true;
            return;
          }

          if (firstRowKey == null) {
            firstRowKey = pivotRow.getKey();
          }

          if (firstRowKey.equals(pivotRow.getKey())) {
            pivotRow = pivotScanner.nextRow();
            if (pivotRow == null) {
              end = true;
              return;
            }
          }
          pivotScanner.close();
          rowFilter.setStartRowKey(firstRowKey);
          pivotScanner =
              ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
        } else {
          firstRowKey = startRowKey;
        }
      } catch (Exception e) {
        if (pivotScanner != null) {
          pivotScanner.close();
        }
        throw e;
      }

      RowFilter joinRowFilter = inputTableInfo.getJoinRowFilter();

      if (mergeEvaluator != null) {
        if (!firstRowKey.equals(Row.Key.MIN_KEY)) {
          joinRowFilter.setStartRowKey(mergeEvaluator.parseTargetRowKey(firstRowKey, 0));
        } else {
          joinRowFilter.setStartRowKey(Row.Key.MIN_KEY);
        }
        if (!rowFilter.getEndRowKey().equals(Row.Key.MAX_KEY)) {
          joinRowFilter.setEndRowKey(mergeEvaluator.parseTargetRowKey(rowFilter.getEndRowKey(), 0));
        } else {
          joinRowFilter.setEndRowKey(Row.Key.MAX_KEY);
        }
      } else {
        joinRowFilter.setStartRowKey(firstRowKey);
        joinRowFilter.setEndRowKey(rowFilter.getEndRowKey());
      }

      reporter.setStatus(
          inputTableInfo.getTableName()
              + ":"
              + startRowKey
              + " ~ "
              + endRowKey
              + ", "
              + inputTableInfo.getJoinTableName()
              + ":"
              + joinRowFilter.getStartRowKey()
              + " ~ "
              + joinRowFilter.getEndRowKey());

      // pivot table의 startRow, endRow에 대응하는 target table의 scanner 생성
      CTable targetTable = CTable.openTable(conf, inputTableInfo.getJoinTableName());
      targetScanner =
          ScannerFactory.openScanner(targetTable, joinRowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);

      this.scanner =
          new MergeScanner(new TableScanner[] {pivotScanner, targetScanner}, mergeEvaluator);
    } catch (Exception e) {
      if (targetScanner != null) {
        targetScanner.close();
      }
      IOException err = new IOException(e.getMessage());
      err.initCause(e);
      throw err;
    }
  }