private static Long latestVersion(Hashtable<String, String> config, dbutil db_util)
throws Exception {
if (!config.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
return new Long(config.get("timestamp_stop"));
}
String rowName = config.get("file_id") + config.get("run_id") + "_";
if (config.get("task_id") != "") {
try {
rowName = rowName + String.format("%04d", new Integer(config.get("task_id")));
} catch (NumberFormatException E) {
rowName = rowName + config.get("task_id");
}
}
Get timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
Result timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
KeyValue tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
if (tsKv == null) {
rowName = config.get("file_id") + "_";
timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
}
if (tsKv == null) {
return new Long(Integer.MAX_VALUE);
}
Long latestVersion = new Long(tsKv.getTimestamp());
return latestVersion;
}
private static Put clonePut(Put put, long startId, boolean locking, byte[] columnsWritten) {
Put ret = new Put(put.getRow());
Map<byte[], List<KeyValue>> families = put.getFamilyMap();
Columns cols = new Columns(columnsWritten);
for (byte[] family : families.keySet()) {
List<KeyValue> columns = families.get(family);
Iterator<KeyValue> it = columns.iterator();
while (it.hasNext()) {
KeyValue kv = it.next();
// byte[] column = DominoConst.getColumnKey(kv.getQualifier(), startId);
byte[] qualifier = kv.getQualifier();
ret.add(family, qualifier, startId, kv.getValue());
cols.add(family, qualifier);
}
}
Map<String, byte[]> attributes = put.getAttributesMap();
for (String key : attributes.keySet()) {
ret.setAttribute(key, attributes.get(key));
}
byte[] state = new byte[1 + Bytes.SIZEOF_LONG];
state[0] = locking ? DominoConst.S_STATEFUL_BYTE : DominoConst.S_STATELESS_BYTE;
Bytes.putLong(state, 1, startId);
ret.add(DominoConst.INNER_FAMILY, DominoConst.COLUMNS_COL, startId, cols.toByteArray());
ret.add(DominoConst.INNER_FAMILY, DominoConst.STATUS_COL, startId, state);
return ret;
}
// @Override
// protected void map(ImmutableBytesWritable key, Text value, Context context) throws
// IOException, InterruptedException {
// Text combinedKeyValue = new Text();
// //the structure is key###value
// combinedKeyValue.set(Bytes.toString(key.get()) + "###" + value.toString());
// context.write(one, combinedKeyValue);
// }
@Override
protected void map(ImmutableBytesWritable key, Result columns, Context context)
throws IOException, InterruptedException {
Text combinedKeyValue = new Text();
// the structure is key###value
String value = null;
try {
for (KeyValue kv : columns.list()) {
byte[] gmmData = kv.getValue();
String gmmString = Bytes.toStringBinary(gmmData);
// /* just for checking that gmm is correctly constructed
MixtureModel m = null;
m = (MixtureModel) ObjectAndByte.byteArrayToObject(Bytes.toBytesBinary(gmmString));
System.out.println("m.size:" + m.size);
// */
combinedKeyValue.set(Bytes.toString(key.get()) + "###" + gmmString);
context.write(one, combinedKeyValue);
// context.write(key, new Text(gmmString));
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Transactional version of {@link HTable#put(Put)}
*
* @param transactionState Identifier of the transaction
* @see HTable#put(Put)
* @throws IOException
*/
public void put(TransactionState transactionState, Put put)
throws IOException, IllegalArgumentException {
final long startTimestamp = transactionState.getStartTimestamp();
// byte[] startTSBytes = Bytes.toBytes(startTimestamp);
// create put with correct ts
final Put tsput = new Put(put.getRow(), startTimestamp);
Map<byte[], List<KeyValue>> kvs = put.getFamilyMap();
for (List<KeyValue> kvl : kvs.values()) {
for (KeyValue kv : kvl) {
// int tsOffset = kv.getTimestampOffset();
// System.arraycopy(startTSBytes, 0, kv.getBuffer(), tsOffset,
// Bytes.SIZEOF_LONG);
tsput.add(
new KeyValue(
kv.getRow(), kv.getFamily(), kv.getQualifier(), startTimestamp, kv.getValue()));
}
}
// should add the table as well
transactionState.addRow(new RowKeyFamily(put.getRow(), getTableName(), put.getFamilyMap()));
put(tsput);
// super.getConnection().getRegionServerWithRetries(
// new ServerCallable<Boolean>(super.getConnection(), super.getTableName(),
// put.getRow()) {
// public Boolean call() throws IOException {
// server.put(location.getRegionInfo().getRegionName(), tsput);
// return true;
// }
// });
}
/** @return string */
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("row=");
sb.append(Bytes.toString(this.row));
sb.append(", ts=");
sb.append(this.ts);
sb.append(", families={");
boolean moreThanOne = false;
for (Map.Entry<byte[], List<KeyValue>> entry : this.familyMap.entrySet()) {
if (moreThanOne) {
sb.append(", ");
} else {
moreThanOne = true;
}
sb.append("(family=");
sb.append(Bytes.toString(entry.getKey()));
sb.append(", keyvalues=(");
boolean moreThanOneB = false;
for (KeyValue kv : entry.getValue()) {
if (moreThanOneB) {
sb.append(", ");
} else {
moreThanOneB = true;
}
sb.append(kv.toString());
}
sb.append(")");
}
sb.append("}");
return sb.toString();
}
public static List<String> getFreeBooks() throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.EQUAL,
Bytes.toBytes("0.00"));
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
/*
* for (KeyValue kv : r.raw()) { System.out.println(String.format(
* "row:%s, family:%s, qualifier:%s, qualifiervalue:%s, timestamp:%s."
* , Bytes.toString(kv.getRow()), Bytes.toString(kv.getFamily()),
* Bytes.toString(kv.getQualifier()), Bytes.toString(kv.getValue()),
* kv.getTimestamp())); }
*/
System.out.println(str);
list.add(str);
}
return list;
}
/**
* This utility method creates a list of Thrift TRowResult "struct" based on an Hbase RowResult
* object. The empty list is returned if the input is null.
*
* @param in Hbase RowResult object
* @param sortColumns This boolean dictates if row data is returned in a sorted order sortColumns
* = True will set TRowResult's sortedColumns member which is an ArrayList of TColumn struct
* sortColumns = False will set TRowResult's columns member which is a map of columnName and
* TCell struct
* @return Thrift TRowResult array
*/
public static List<TRowResult> rowResultFromHBase(Result[] in, boolean sortColumns) {
List<TRowResult> results = new ArrayList<TRowResult>();
for (Result result_ : in) {
if (result_ == null || result_.isEmpty()) {
continue;
}
TRowResult result = new TRowResult();
result.row = ByteBuffer.wrap(result_.getRow());
if (sortColumns) {
result.sortedColumns = new ArrayList<TColumn>();
for (Cell kv : result_.rawCells()) {
result.sortedColumns.add(
new TColumn(
ByteBuffer.wrap(
KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp())));
}
} else {
result.columns = new TreeMap<ByteBuffer, TCell>();
for (Cell kv : result_.rawCells()) {
result.columns.put(
ByteBuffer.wrap(
KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp()));
}
}
results.add(result);
}
return results;
}
protected boolean skipKVsNewerThanReadpoint() throws IOException {
long readPoint = MultiVersionConsistencyControl.getThreadReadPoint();
// We want to ignore all key-values that are newer than our current
// readPoint
while (enforceMVCC && cur != null && (cur.getMemstoreTS() > readPoint)) {
hfs.next();
cur = hfs.getKeyValue();
}
if (cur == null) {
close();
return false;
}
// For the optimisation in HBASE-4346, we set the KV's memstoreTS to
// 0, if it is older than all the scanners' read points. It is possible
// that a newer KV's memstoreTS was reset to 0. But, there is an
// older KV which was not reset to 0 (because it was
// not old enough during flush). Make sure that we set it correctly now,
// so that the comparision order does not change.
if (cur.getMemstoreTS() <= readPoint) {
cur.setMemstoreTS(0);
}
return true;
}
private static boolean hasFile(
dbutil db_util, FileSystem fs, String db_name, String file_id, String file_path)
throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names == null) {
return false;
}
String all_files = new String(file_names.getValue());
String[] files = all_files.split("\n");
for (String line : files) {
if (line.equals(file_path)) {
if (fs.globStatus(new Path(line + "*")).length == 0) {
Put new_put = new Put(file_id.getBytes());
new_put.add(
"d".getBytes(),
"filenames".getBytes(),
all_files.replace(file_path + "\n", "").getBytes());
db_util.doPut(db_name, new_put);
return false;
}
return true;
}
}
return false;
}
private static boolean putFileEntry(
dbutil db_util,
FileSystem fs,
String db_name,
String file_id,
String file_path,
String source)
throws Exception {
String all_paths = file_path;
if (hasFile(db_util, fs, db_name, file_id, file_path)) {
logger.debug("File already found, putFileEntry aborting");
return false;
} else {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names != null) {
String paths = new String(file_names.getValue());
all_paths = paths + "\n" + file_path;
}
}
Put file_id_put = new Put(file_id.getBytes());
file_id_put.add("d".getBytes(), "source".getBytes(), source.getBytes());
if (!source.equals("fullfile")) {
file_id_put.add("d".getBytes(), "filenames".getBytes(), all_paths.getBytes());
}
db_util.doPut(db_name, file_id_put);
return true;
}
/**
* Imperfect estimate of row size given a PTable TODO: keep row count in stats table and use total
* size / row count instead
*
* @param table
* @return estimate of size in bytes of a row
*/
public static long estimateRowSize(PTable table) {
int keyLength = estimateKeyLength(table);
long rowSize = 0;
for (PColumn column : table.getColumns()) {
if (!SchemaUtil.isPKColumn(column)) {
PDataType type = column.getDataType();
Integer maxLength = column.getMaxLength();
int valueLength =
!type.isFixedWidth()
? VAR_KV_LENGTH_ESTIMATE
: maxLength == null ? type.getByteSize() : maxLength;
rowSize +=
KeyValue.getKeyValueDataStructureSize(
keyLength,
column.getFamilyName().getBytes().length,
column.getName().getBytes().length,
valueLength);
}
}
// Empty key value
rowSize +=
KeyValue.getKeyValueDataStructureSize(
keyLength,
getEmptyColumnFamily(table).length,
QueryConstants.EMPTY_COLUMN_BYTES.length,
0);
return rowSize;
}
public static List<String> getBooksbyPrice(String min, String max) throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
FilterList filterList = new FilterList();
Filter maxFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.GREATER_OR_EQUAL,
Bytes.toBytes(min));
Filter minFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.LESS_OR_EQUAL,
Bytes.toBytes(max));
filterList.addFilter(maxFilter);
filterList.addFilter(minFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
System.out.println(str);
list.add(str);
}
return list;
}
public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
Scan scan = new Scan();
scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
List<String> result = new ArrayList<String>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String clusterId = Bytes.toString(kv.getRow());
String[] parts = clusterId.split(":");
double cLat = Double.parseDouble(parts[0]);
double cLon = Double.parseDouble(parts[1]);
double dx = Double.parseDouble(parts[2]);
double dy = Double.parseDouble(parts[3]);
double distx = lat - cLat;
double disty = lon - cLon;
if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
// get stations in cluster
result.add(Bytes.toString(kv.getQualifier()));
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
@Override
public Map<String, Integer> getAvailableBikesFromAPoint_Schema2(Scan scan) throws IOException {
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
Map<String, Integer> result = new HashMap<String, Integer>();
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Integer free = Integer.parseInt(value.split(";")[0]);
result.put(stationId, free);
/*if(result.containsKey(stationId)){
result.put(stationId, free + result.get(stationId));
}else{
result.put(stationId, free);
}*/
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
public static String get(String keyspace, String rowKey, String column, long timestamp)
throws Exception {
String columnValue = null;
HTable htable = new HTable(keyspace);
Get get = new Get(rowKey.getBytes());
get = get.setTimeStamp(timestamp);
get = get.setMaxVersions();
Result res = htable.get(get);
KeyValue[] data = res.raw();
for (int i = 0; i < data.length; i++) {
KeyValue d = data[i];
String family = new String(data[i].getFamily());
String qualifier = new String(data[i].getQualifier());
if (qualifier.toLowerCase().equals(column.toLowerCase())) {
columnValue = new String(d.getValue());
System.out.println(
data[i].toString()
+ " Family:"
+ family
+ " Qualifier:"
+ qualifier
+ " Value:"
+ columnValue);
break;
}
}
return columnValue;
}
/**
* make a general method that takes a pair of lat/lon and a radius and give a boolean whether it
* was in or out.
*
* @throws IOException
*/
@Override
public Map<String, Integer> getAvailableBikesFromAPoint(
double lat, double lon, double radius, Get get) throws IOException {
Result r = ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().get(get, null);
log.debug("r is " + r);
log.debug(r.getMap().toString());
Map<String, Integer> result = new HashMap<String, Integer>();
try {
String s = null, latStr = null, lonStr = null;
for (KeyValue kv : r.raw()) {
s = Bytes.toString(kv.getValue());
log.debug("cell value is: " + s);
String[] sArr = s.split(BIXI_DELIMITER); // array of key=value pairs
latStr = sArr[3];
lonStr = sArr[4];
latStr = latStr.substring(latStr.indexOf("=") + 1);
lonStr = lonStr.substring(lonStr.indexOf("=") + 1);
log.debug("lon/lat values are: " + lonStr + "; " + latStr);
double distance =
giveDistance(Double.parseDouble(latStr), Double.parseDouble(lonStr), lat, lon) - radius;
log.debug("distance is : " + distance);
if (distance < 0) { // add it
result.put(sArr[0], getFreeBikes(kv));
}
}
} finally {
}
return result;
}
@Override
public Map<String, TotalNum> getTotalUsage_Schema2(Scan scan) throws IOException {
// System.err.println("scanning");
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, TotalNum> result = new HashMap<String, TotalNum>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Long usage = Long.parseLong(value.split(";")[1]);
if (result.containsKey(stationId)) {
TotalNum tn = result.get(stationId);
tn.add(usage);
result.put(stationId, tn);
} else {
TotalNum tn = new TotalNum();
tn.add(usage);
result.put(stationId, tn);
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
/** {@inheritDoc} */
@Override
public void write(HFileKeyValue entry, NullWritable unused) throws IOException {
final KeyValue kv = entry.getKeyValue();
kv.updateLatestStamp(mLatestTimestampBytes);
final long recordLength = kv.getLength();
if (mCurrentHFileSize + recordLength >= mMaxFileSizeBytes) {
// We can't fit this record in the current HFile without exceeding the max file size.
if (Arrays.equals(mCurrentRow, kv.getRow())) {
// But we're still adding data for a single row, so we can't close this HFile yet.
LOG.debug("Reached max HFile size, but waiting to finish this row before closing.");
} else {
// Close it and open a new one.
closeWriter(mWriter);
mWriter = openNewWriter();
}
}
mWriter.append(kv);
mTimeRangeTracker.includeTimestamp(kv);
mCurrentHFileSize += recordLength;
// Remember the row so we know when we are transitioning.
mCurrentRow = kv.getRow();
}
private void performMerge(List<StoreFileScanner> scanners, HStore store, StoreFile.Writer writer)
throws IOException {
InternalScanner scanner = null;
try {
Scan scan = new Scan();
// Include deletes
scanner =
new StoreScanner(
store,
store.scanInfo,
scan,
scanners,
ScanType.MAJOR_COMPACT,
Long.MIN_VALUE,
Long.MIN_VALUE);
ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
while (scanner.next(kvs) || kvs.size() != 0) {
numKV.addAndGet(kvs.size());
for (KeyValue kv : kvs) {
totalBytes.addAndGet(kv.getLength());
writer.append(kv);
}
kvs.clear();
}
} finally {
if (scanner != null) scanner.close();
}
}
/*
* Test that {@link HFileOutputFormat2} creates an HFile with TIMERANGE
* metadata used by time-restricted scans.
*/
@Test
public void test_TIMERANGE() throws Exception {
Configuration conf = new Configuration(this.util.getConfiguration());
RecordWriter<ImmutableBytesWritable, Cell> writer = null;
TaskAttemptContext context = null;
Path dir = util.getDataTestDir("test_TIMERANGE_present");
LOG.info("Timerange dir writing to dir: " + dir);
try {
// build a record writer using HFileOutputFormat2
Job job = new Job(conf);
FileOutputFormat.setOutputPath(job, dir);
context = createTestTaskAttemptContext(job);
HFileOutputFormat2 hof = new HFileOutputFormat2();
writer = hof.getRecordWriter(context);
// Pass two key values with explicit times stamps
final byte[] b = Bytes.toBytes("b");
// value 1 with timestamp 2000
KeyValue kv = new KeyValue(b, b, b, 2000, b);
KeyValue original = kv.clone();
writer.write(new ImmutableBytesWritable(), kv);
assertEquals(original, kv);
// value 2 with timestamp 1000
kv = new KeyValue(b, b, b, 1000, b);
original = kv.clone();
writer.write(new ImmutableBytesWritable(), kv);
assertEquals(original, kv);
// verify that the file has the proper FileInfo.
writer.close(context);
// the generated file lives 1 directory down from the attempt directory
// and is the only file, e.g.
// _attempt__0000_r_000000_0/b/1979617994050536795
FileSystem fs = FileSystem.get(conf);
Path attemptDirectory = hof.getDefaultWorkFile(context, "").getParent();
FileStatus[] sub1 = fs.listStatus(attemptDirectory);
FileStatus[] file = fs.listStatus(sub1[0].getPath());
// open as HFile Reader and pull out TIMERANGE FileInfo.
HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), new CacheConfig(conf), conf);
Map<byte[], byte[]> finfo = rd.loadFileInfo();
byte[] range = finfo.get("TIMERANGE".getBytes());
assertNotNull(range);
// unmarshall and check values.
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
Writables.copyWritable(range, timeRangeTracker);
LOG.info(
timeRangeTracker.getMinimumTimestamp() + "...." + timeRangeTracker.getMaximumTimestamp());
assertEquals(1000, timeRangeTracker.getMinimumTimestamp());
assertEquals(2000, timeRangeTracker.getMaximumTimestamp());
rd.close();
} finally {
if (writer != null && context != null) writer.close(context);
dir.getFileSystem(conf).delete(dir, true);
}
}
@Override
public Map<String, Integer> giveAvailableBikes(
long milliseconds, List<String> stationIds, Scan scan) throws IOException {
// scan has set the time stamp accordingly, i.e., the start and end row of
// the scan.
for (String qualifier : stationIds) {
log.debug("adding qualifier: " + qualifier);
scan.addColumn(colFamily, qualifier.getBytes());
}
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, Integer> result = new HashMap<String, Integer>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
// log.debug("got a kv: " + kv);
int availBikes = getFreeBikes(kv);
String id = Bytes.toString(kv.getQualifier());
// log.debug("result to be added is: " + availBikes + " id: " + id);
result.put(id, availBikes);
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
public static HBaseTuple getHBaseTuple(KeyValue kv) {
HBaseTuple tuple = new HBaseTuple();
tuple.setRow(new String(kv.getRow()));
tuple.setColFamily(new String(kv.getFamily()));
tuple.setColName(new String(kv.getQualifier()));
tuple.setColValue(new String(kv.getValue()));
return tuple;
}
private static boolean containsStatus(Result r, long startId) {
List<KeyValue> status = r.getColumn(DominoConst.INNER_FAMILY, DominoConst.STATUS_COL);
if (status == null) return false;
for (KeyValue kv : status) {
if (kv.getTimestamp() == startId) return true;
}
return false;
}
public boolean addDeleted(KeyValue kv) {
if (kv.getValue().length == 0) {
deletedColumns.put(
new ByteArray(Bytes.add(kv.getFamily(), kv.getQualifier())), kv.getTimestamp());
return true;
}
return false;
}
@SuppressWarnings("deprecation")
@Override
protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
buffer.putInt(kv.getLength());
buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
}
/**
* Performs the compaction.
*
* @param scanner Where to read from.
* @param writer Where to write to.
* @param smallestReadPoint Smallest read point.
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
* @return Whether compaction ended; false if it was interrupted for some reason.
*/
protected boolean performCompaction(
InternalScanner scanner, CellSink writer, long smallestReadPoint, boolean cleanSeqId)
throws IOException {
int bytesWritten = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> kvs = new ArrayList<Cell>();
int closeCheckInterval = HStore.getCloseCheckInterval();
long lastMillis;
if (LOG.isDebugEnabled()) {
lastMillis = System.currentTimeMillis();
} else {
lastMillis = 0;
}
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
// output to writer:
for (Cell c : kvs) {
KeyValue kv = KeyValueUtil.ensureKeyValue(c);
if (cleanSeqId && kv.getSequenceId() <= smallestReadPoint) {
kv.setSequenceId(0);
}
writer.append(kv);
++progress.currentCompactedKVs;
progress.totalCompactedSize += kv.getLength();
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > closeCheckInterval) {
// Log the progress of long running compactions every minute if
// logging at DEBUG level
if (LOG.isDebugEnabled()) {
long now = System.currentTimeMillis();
if ((now - lastMillis) >= 60 * 1000) {
LOG.debug(
"Compaction progress: "
+ progress
+ String.format(
", rate=%.2f kB/sec",
(bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
lastMillis = now;
}
}
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
return false;
}
}
}
}
kvs.clear();
} while (hasMore);
progress.complete();
return true;
}
private static String getSource(dbutil db_util, String db_name, String file_id) throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_source = file_result.getColumnLatest("d".getBytes(), "source".getBytes());
if (file_source == null) {
return "local";
}
return new String(file_source.getValue());
}
/**
* Maps the data.
*
* @param row The current table row key.
* @param values The columns.
* @param context The current context.
* @throws IOException When something is broken with the data.
* @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
* org.apache.hadoop.mapreduce.Mapper.Context)
*/
@Override
public void map(ImmutableBytesWritable row, Result values, Context context) throws IOException {
for (KeyValue value : values.list()) {
if (value.getValue().length > 0) {
context.getCounter(Counters.ROWS).increment(1);
break;
}
}
}
/**
* Sets the row this instance holds in RAM using a row from a scanner.
*
* @param row The compacted HBase row to set.
* @throws IllegalStateException if this method was already called.
*/
void setRow(final KeyValue row) {
if (this.key != null) {
throw new IllegalStateException("setRow was already called on " + this);
}
this.key = row.getRow();
this.qualifiers = row.getQualifier();
this.values = row.getValue();
}
/**
* Flushes the snapshot of the memstore. Flushes the mob data to the mob files, and flushes the
* name of these mob files to HBase.
*
* @param snapshot The snapshot of the memstore.
* @throws IOException
*/
private void internalFlushCache(final MemStoreSnapshot snapshot) throws IOException {
if (snapshot.getCellsCount() == 0) {
return;
}
// generate the files into a temp directory.
String tempPathString = context.getConfiguration().get(SweepJob.WORKING_FILES_DIR_KEY);
StoreFile.Writer mobFileWriter =
MobUtils.createWriter(
conf,
fs,
hcd,
partitionId.getDate(),
new Path(tempPathString),
snapshot.getCellsCount(),
hcd.getCompactionCompression(),
partitionId.getStartKey(),
cacheConfig,
cryptoContext);
String relativePath = mobFileWriter.getPath().getName();
LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());
byte[] referenceValue = Bytes.toBytes(relativePath);
KeyValueScanner scanner = snapshot.getScanner();
Cell cell = null;
while (null != (cell = scanner.next())) {
mobFileWriter.append(cell);
}
scanner.close();
// Write out the log sequence number that corresponds to this output
// hfile. The hfile is current up to and including logCacheFlushId.
mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
mobFileWriter.close();
MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
context.getCounter(SweepCounter.FILE_AFTER_MERGE_OR_CLEAN).increment(1);
// write reference/fileName back to the store files of HBase.
scanner = snapshot.getScanner();
scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
cell = null;
Tag tableNameTag =
new ArrayBackedTag(
TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
long updatedCount = 0;
while (null != (cell = scanner.next())) {
KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
Put put =
new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
put.add(reference);
table.mutate(put);
updatedCount++;
}
table.flush();
context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
scanner.close();
}
