protected void doReconstructionLog( final Path oldCoreLogFile, final long minSeqId, final long maxSeqId, final Progressable reporter) throws UnsupportedEncodingException, IOException { Path trxPath = new Path(oldCoreLogFile.getParent(), THLog.HREGION_OLD_THLOGFILE_NAME); // We can ignore doing anything with the Trx Log table, it is // not-transactional. if (super.getTableDesc().getNameAsString().equals(HBaseBackedTransactionLogger.TABLE_NAME)) { return; } THLogRecoveryManager recoveryManager = new THLogRecoveryManager(this); Map<Long, WALEdit> commitedTransactionsById = recoveryManager.getCommitsFromLog(trxPath, minSeqId, reporter); if (commitedTransactionsById != null && commitedTransactionsById.size() > 0) { LOG.debug("found " + commitedTransactionsById.size() + " COMMITED transactions to recover."); for (Entry<Long, WALEdit> entry : commitedTransactionsById.entrySet()) { LOG.debug( "Writing " + entry.getValue().size() + " updates for transaction " + entry.getKey()); WALEdit b = entry.getValue(); for (KeyValue kv : b.getKeyValues()) { // FIXME need to convert these into puts and deletes. Not sure this is // the write way. // Could probably combine multiple KV's into single put/delete. // Also timestamps? if (kv.getType() == KeyValue.Type.Put.getCode()) { Put put = new Put(); put.add(kv); super.put(put); } else if (kv.isDelete()) { Delete del = new Delete(kv.getRow()); if (kv.isDeleteFamily()) { del.deleteFamily(kv.getFamily()); } else if (kv.isDeleteType()) { del.deleteColumn(kv.getFamily(), kv.getQualifier()); } } } } LOG.debug("Flushing cache"); // We must trigger a cache flush, // otherwise we will would ignore the log on subsequent failure if (!super.flushcache()) { LOG.warn("Did not flush cache"); } } }
/** * Check if the specified KeyValue buffer has been deleted by a previously seen delete. * * @param kv * @param ds * @return True is the specified KeyValue is deleted, false if not */ public boolean isDeleted(final KeyValue kv, final NavigableSet<KeyValue> ds) { if (deletes == null || deletes.isEmpty()) return false; for (KeyValue d : ds) { long kvts = kv.getTimestamp(); long dts = d.getTimestamp(); if (d.isDeleteFamily()) { if (kvts <= dts) return true; continue; } // Check column int ret = Bytes.compareTo( kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), d.getBuffer(), d.getQualifierOffset(), d.getQualifierLength()); if (ret <= -1) { // This delete is for an earlier column. continue; } else if (ret >= 1) { // Beyond this kv. break; } // Check Timestamp if (kvts > dts) return false; // Check Type switch (KeyValue.Type.codeToType(d.getType())) { case Delete: return kvts == dts; case DeleteColumn: return true; default: continue; } } return false; }
/** * Pretend we have done a seek but don't do it yet, if possible. The hope is that we find * requested columns in more recent files and won't have to seek in older files. Creates a fake * key/value with the given row/column and the highest (most recent) possible timestamp we might * get from this file. When users of such "lazy scanner" need to know the next KV precisely (e.g. * when this scanner is at the top of the heap), they run {@link #enforceSeek()}. * * <p>Note that this function does guarantee that the current KV of this scanner will be advanced * to at least the given KV. Because of this, it does have to do a real seek in cases when the * seek timestamp is older than the highest timestamp of the file, e.g. when we are trying to seek * to the next row/column and use OLDEST_TIMESTAMP in the seek key. */ @Override public boolean requestSeek(KeyValue kv, boolean forward, boolean useBloom) throws IOException { if (kv.getFamilyLength() == 0) { useBloom = false; } boolean haveToSeek = true; if (useBloom) { // check ROWCOL Bloom filter first. if (reader.getBloomFilterType() == StoreFile.BloomType.ROWCOL) { haveToSeek = reader.passesGeneralBloomFilter( kv.getBuffer(), kv.getRowOffset(), kv.getRowLength(), kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); } else if (this.matcher != null && !matcher.hasNullColumnInQuery() && kv.isDeleteFamily()) { // if there is no such delete family kv in the store file, // then no need to seek. haveToSeek = reader.passesDeleteFamilyBloomFilter( kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()); } } delayedReseek = forward; delayedSeekKV = kv; if (haveToSeek) { // This row/column might be in this store file (or we did not use the // Bloom filter), so we still need to seek. realSeekDone = false; long maxTimestampInFile = reader.getMaxTimestamp(); long seekTimestamp = kv.getTimestamp(); if (seekTimestamp > maxTimestampInFile) { // Create a fake key that is not greater than the real next key. // (Lower timestamps correspond to higher KVs.) // To understand this better, consider that we are asked to seek to // a higher timestamp than the max timestamp in this file. We know that // the next point when we have to consider this file again is when we // pass the max timestamp of this file (with the same row/column). cur = kv.createFirstOnRowColTS(maxTimestampInFile); } else { // This will be the case e.g. when we need to seek to the next // row/column, and we don't know exactly what they are, so we set the // seek key's timestamp to OLDEST_TIMESTAMP to skip the rest of this // row/column. enforceSeek(); } return cur != null; } // Multi-column Bloom filter optimization. // Create a fake key/value, so that this scanner only bubbles up to the top // of the KeyValueHeap in StoreScanner after we scanned this row/column in // all other store files. The query matcher will then just skip this fake // key/value and the store scanner will progress to the next column. This // is obviously not a "real real" seek, but unlike the fake KV earlier in // this method, we want this to be propagated to ScanQueryMatcher. cur = kv.createLastOnRowCol(); realSeekDone = true; return true; }