public void map(
WritableComparable key,
Writable value,
OutputCollector<WritableComparable, Writable> collector,
Reporter reporter)
throws IOException {
LOG.info("Start Map");
if (err != null) {
throw err;
}
DecimalFormat df = new DecimalFormat("00000000000000000000");
collector.collect(new Text(tableName), new Text(""));
for (long i = 0; i < 50000; i++) {
long randNum = rand.nextLong();
Row.Key rowKey = new Row.Key(df.format(randNum));
Row row = new Row(rowKey);
row.addCell("Col1", new Cell(Cell.Key.EMPTY_KEY, this.data));
ctable.put(row);
if (i % 1000 == 0) {
reporter.progress();
}
if (i % 10000 == 0) {
LOG.info("uploaded: " + i);
}
collector.collect(new Text(df.format(randNum)), new Text(""));
}
LOG.info("End Map");
}
public void map(
WritableComparable key,
Writable value,
OutputCollector<WritableComparable, Writable> collector,
Reporter reporter)
throws IOException {
if (err != null) {
throw err;
}
String valueStr = value.toString().trim();
if (valueStr.indexOf("\t") > 0) {
valueStr = valueStr.substring(0, valueStr.indexOf("\t"));
}
if (this.ctable == null) {
this.ctable = CTable.openTable(conf, valueStr);
if (this.ctable == null) {
throw new IOException("No table:" + valueStr);
}
return;
}
Row.Key rowKey = new Row.Key(valueStr);
Row row = ctable.get(rowKey);
if (row == null) {
throw new IOException("No Data:[" + rowKey + "]");
}
Cell cell = row.getFirst("Col1");
if (cell == null) {
throw new IOException("No Cell Data:[" + rowKey + "]");
}
if (count % 10000 == 0) {
LOG.info("read: " + count + ", last row key[" + rowKey + "]");
}
count++;
}
public TableJoinRecordReader(
JobConf jobConf, CloudataConf conf, TableSplit tableSplit, Reporter reporter)
throws IOException {
this.conf = conf;
String mergeEvaluatorClass = tableSplit.getInputTableInfo().getMergeEvaluatorClass();
MergeEvaluator mergeEvaluator = null;
if (mergeEvaluatorClass != null && mergeEvaluatorClass.length() > 0) {
try {
mergeEvaluator = (MergeEvaluator) Class.forName(mergeEvaluatorClass).newInstance();
} catch (Exception e) {
LOG.error("mergeEvaluator:" + mergeEvaluatorClass + "," + e.getMessage());
IOException err = new IOException(e.getMessage() + ":" + mergeEvaluatorClass);
err.initCause(e);
throw err;
}
}
RowFilter splitRowFilter = tableSplit.getRowFilter();
InputTableInfo inputTableInfo = tableSplit.getInputTableInfo();
this.startRowKey = splitRowFilter.getStartRowKey();
this.endRowKey = splitRowFilter.getEndRowKey();
RowFilter rowFilter = inputTableInfo.getRowFilter();
rowFilter.setStartRowKey(startRowKey);
rowFilter.setEndRowKey(endRowKey);
CTable ctable = CTable.openTable(conf, inputTableInfo.getTableName());
TableScanner pivotScanner = null;
TableScanner targetScanner = null;
try {
pivotScanner =
ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
Row.Key firstRowKey = null;
try {
// 첫번째 Tablet이 아닌 경우에는 첫번째 row는 무시한다.
if (!startRowKey.equals(Row.Key.MIN_KEY)) {
Row pivotRow = pivotScanner.nextRow();
if (pivotRow == null) {
end = true;
return;
}
if (firstRowKey == null) {
firstRowKey = pivotRow.getKey();
}
if (firstRowKey.equals(pivotRow.getKey())) {
pivotRow = pivotScanner.nextRow();
if (pivotRow == null) {
end = true;
return;
}
}
pivotScanner.close();
rowFilter.setStartRowKey(firstRowKey);
pivotScanner =
ScannerFactory.openScanner(ctable, rowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
} else {
firstRowKey = startRowKey;
}
} catch (Exception e) {
if (pivotScanner != null) {
pivotScanner.close();
}
throw e;
}
RowFilter joinRowFilter = inputTableInfo.getJoinRowFilter();
if (mergeEvaluator != null) {
if (!firstRowKey.equals(Row.Key.MIN_KEY)) {
joinRowFilter.setStartRowKey(mergeEvaluator.parseTargetRowKey(firstRowKey, 0));
} else {
joinRowFilter.setStartRowKey(Row.Key.MIN_KEY);
}
if (!rowFilter.getEndRowKey().equals(Row.Key.MAX_KEY)) {
joinRowFilter.setEndRowKey(mergeEvaluator.parseTargetRowKey(rowFilter.getEndRowKey(), 0));
} else {
joinRowFilter.setEndRowKey(Row.Key.MAX_KEY);
}
} else {
joinRowFilter.setStartRowKey(firstRowKey);
joinRowFilter.setEndRowKey(rowFilter.getEndRowKey());
}
reporter.setStatus(
inputTableInfo.getTableName()
+ ":"
+ startRowKey
+ " ~ "
+ endRowKey
+ ", "
+ inputTableInfo.getJoinTableName()
+ ":"
+ joinRowFilter.getStartRowKey()
+ " ~ "
+ joinRowFilter.getEndRowKey());
// pivot table의 startRow, endRow에 대응하는 target table의 scanner 생성
CTable targetTable = CTable.openTable(conf, inputTableInfo.getJoinTableName());
targetScanner =
ScannerFactory.openScanner(targetTable, joinRowFilter, TableScanner.SCANNER_OPEN_TIMEOUT);
this.scanner =
new MergeScanner(new TableScanner[] {pivotScanner, targetScanner}, mergeEvaluator);
} catch (Exception e) {
if (targetScanner != null) {
targetScanner.close();
}
IOException err = new IOException(e.getMessage());
err.initCause(e);
throw err;
}
}