public boolean fetchLine(Line line) throws IOException {
if (null == this.rs) {
throw new IllegalStateException("HBase Client try to fetch data failed .");
}
Result result = this.rs.next();
if (null == result) {
return false;
}
if (this.maxversion == -1) { // 多版本记录,按照每个字段的设置获取不同版本数据
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> nMap = result.getMap();
this.p = new Put(result.getRow());
this.p.setDurability(Durability.SKIP_WAL);
for (int i = 0; i < this.families.length; i++) {
NavigableMap<Long, byte[]> vmap =
nMap.get(this.families[i].getBytes()).get(this.columns[i].getBytes());
byte[] value = null;
byte[] firstValue = null;
if (vmap == null || vmap.size() == 0) { // 无记录，判断后获取替换的字段值
line.addField(null);
continue;
} else if (vmap.size() > 1) { // 多个版本
if ("1".equalsIgnoreCase(this.column_version[i])) {
Iterator<Map.Entry<Long, byte[]>> iter = vmap.entrySet().iterator();
int id = 0;
while (iter.hasNext()) {
if (id == 0) {
firstValue = iter.next().getValue();
value = firstValue;
} else {
value = iter.next().getValue();
}
id++;
}
if (id > 0) {
this.p.addColumn(families[i].getBytes(), this.columns[i].getBytes(), value);
}
} else { // 取第一个最新版本的值
value = vmap.entrySet().iterator().next().getValue();
}
} else { // 单个版本
value = vmap.entrySet().iterator().next().getValue();
}
if (null == value) {
line.addField(null);
} else {
line.addField(new String(value, encode));
}
}
// 判断是否将hbase值替换和修改
if (ETLStringUtils.isNotEmpty(this.columnProcRule) && this.partRuleId == 1) { // 需要替换字段
if (ETLStringUtils.isEmpty(line.getField(this.partColumnIdx))) { // 需要替换的字段为空，则替换字段
line.addField(line.getField(this.bakPartColumnIdx), this.partColumnIdx);
this.p.addColumn(
families[partColumnIdx].getBytes(),
this.columns[partColumnIdx].getBytes(),
line.getField(this.bakPartColumnIdx).getBytes());
}
}
if (this.p.size() > 0) {
buffer.add(this.p);
}
if (buffer.size() >= BUFFER_LINE) {
htable.put(buffer);
htable.flushCommits();
buffer.clear();
}
} else {
for (int i = 0; i < this.families.length; i++) {
byte[] value = result.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
if (null == value) {
line.addField(null);
} else {
line.addField(new String(value, encode));
}
}
}
//
line.addField(new String(result.getRow(), encode));
return true;
}
/*
* Must be sure that column is in format like 'family: column'
*/
public void prepare(String[] columns) throws IOException {
this.scan = new Scan();
if (this.startKey != null) {
logger.info(String.format("HBaseReader set startkey to %s .", Bytes.toString(this.startKey)));
scan.setStartRow(startKey);
}
if (this.endKey != null) {
logger.info(String.format("HBaseReader set endkey to %s .", Bytes.toString(this.endKey)));
scan.setStopRow(endKey);
}
if (this.minStamp > 0 && this.maxStamp > 0) {
scan.setTimeRange(minStamp, maxStamp);
if (StringUtils.isNotEmpty(primaryKey)) {
String[] cs = StringUtils.split(primaryKey, ":");
scan.addColumn(cs[0].getBytes(), cs[1].getBytes());
}
} else if (ETLStringUtils.isNotEmpty(startDate)
&& ETLStringUtils.isNotEmpty(familyName)
&& ETLStringUtils.isNotEmpty(fieldName)) {
SingleColumnValueFilter scvf =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.GREATER_OR_EQUAL,
new BinaryPrefixComparator(startDate.getBytes()));
scvf.setFilterIfMissing(true);
scvf.setLatestVersionOnly(true);
if (ETLStringUtils.isNotEmpty(endDate)) {
SingleColumnValueFilter scvf1 =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.LESS_OR_EQUAL,
new BinaryPrefixComparator(endDate.getBytes()));
scvf1.setFilterIfMissing(true);
scvf1.setLatestVersionOnly(true);
List<Filter> filters = new ArrayList<Filter>();
filters.add(scvf);
filters.add(scvf1);
FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
scan.setFilter(fl);
} else {
scan.setFilter(scvf);
}
}
this.families = new String[columns.length];
this.columns = new String[columns.length];
int idx = 0;
for (String column : columns) {
this.families[idx] = column.split(":")[0].trim();
this.columns[idx] = column.split(":")[1].trim();
if (!(this.minStamp > 0 && this.maxStamp > 0)) {
scan.addColumn(this.families[idx].getBytes(), this.columns[idx].getBytes());
}
idx++;
}
if ("true".equalsIgnoreCase(this.isSaveOneObj)) {
scan.addColumn(this.oneObjFamilyName.getBytes(), this.oneObjColName.getBytes());
}
if (this.maxversion > 0) {
scan.setMaxVersions(this.maxversion);
} else {
scan.setMaxVersions();
}
htable.setScannerCaching(SCAN_CACHE);
this.rs = htable.getScanner(this.scan);
}