private void flushBufferedData() throws IOException {
if (o == 0) {
return;
}
checksum.reset();
checksum.update(buffer, 0, o);
final int check = (int) checksum.getValue();
int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH);
final int compressMethod;
if (compressedLength >= o) {
compressMethod = COMPRESSION_METHOD_RAW;
compressedLength = o;
System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o);
} else {
compressMethod = COMPRESSION_METHOD_LZ4;
}
compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1);
writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5);
writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9);
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength);
o = 0;
}
private void decompress(byte[] compressed) throws IOException {
// uncompress
validBufferBytes =
info.parameters.sstableCompressor.uncompress(
compressed, 0, compressed.length - checksumBytes.length, buffer, 0);
uncompressedBytes += validBufferBytes;
// validate crc randomly
if (info.parameters.getCrcCheckChance() > FBUtilities.threadLocalRandom().nextDouble()) {
checksum.update(buffer, 0, validBufferBytes);
System.arraycopy(
compressed,
compressed.length - checksumBytes.length,
checksumBytes,
0,
checksumBytes.length);
if (Ints.fromByteArray(checksumBytes) != (int) checksum.getValue())
throw new IOException("CRC unmatched");
// reset checksum object back to the original (blank) state
checksum.reset();
}
// buffer offset is always aligned
bufferOffset = current & ~(buffer.length - 1);
}
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
final DataInputStream in =
new DataInputStream(
SecureIOUtils.openForRead(
new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
try {
final long length = rfs.getFileStatus(indexFileName).getLen();
final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;
buf = ByteBuffer.allocate(size);
if (crc != null) {
crc.reset();
CheckedInputStream chk = new CheckedInputStream(in, crc);
IOUtils.readFully(chk, buf.array(), 0, size);
if (chk.getChecksum().getValue() != in.readLong()) {
throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
}
} else {
IOUtils.readFully(in, buf.array(), 0, size);
}
entries = buf.asLongBuffer();
} finally {
in.close();
}
}
public void writeToFile(Path loc, JobConf job, Checksum crc) throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
CheckedOutputStream chk = null;
final FSDataOutputStream out = rfs.create(loc);
try {
if (crc != null) {
crc.reset();
chk = new CheckedOutputStream(out, crc);
chk.write(buf.array());
out.writeLong(chk.getChecksum().getValue());
} else {
out.write(buf.array());
}
} finally {
if (chk != null) {
chk.close();
} else {
out.close();
}
}
}
public void recover(File file) throws IOException {
logger.info("Replaying " + file.getPath());
final long segment = CommitLogSegment.idFromFilename(file.getName());
RandomAccessReader reader = RandomAccessReader.open(new File(file.getAbsolutePath()), true);
try {
assert reader.length() <= Integer.MAX_VALUE;
int replayPosition;
if (globalPosition.segment < segment) replayPosition = 0;
else if (globalPosition.segment == segment) replayPosition = globalPosition.position;
else replayPosition = (int) reader.length();
if (replayPosition < 0 || replayPosition >= reader.length()) {
// replayPosition > reader.length() can happen if some data gets flushed before it is
// written to the commitlog
// (see https://issues.apache.org/jira/browse/CASSANDRA-2285)
logger.debug("skipping replay of fully-flushed {}", file);
return;
}
reader.seek(replayPosition);
if (logger.isDebugEnabled())
logger.debug("Replaying " + file + " starting at " + reader.getFilePointer());
/* read the logs populate RowMutation and apply */
while (!reader.isEOF()) {
if (logger.isDebugEnabled()) logger.debug("Reading mutation at " + reader.getFilePointer());
long claimedCRC32;
int serializedSize;
try {
// any of the reads may hit EOF
serializedSize = reader.readInt();
if (serializedSize == CommitLog.END_OF_SEGMENT_MARKER) {
logger.debug("Encountered end of segment marker at " + reader.getFilePointer());
break;
}
// RowMutation must be at LEAST 10 bytes:
// 3 each for a non-empty Table and Key (including the
// 2-byte length from writeUTF/writeWithShortLength) and 4 bytes for column count.
// This prevents CRC by being fooled by special-case garbage in the file; see
// CASSANDRA-2128
if (serializedSize < 10) break;
long claimedSizeChecksum = reader.readLong();
checksum.reset();
checksum.update(serializedSize);
if (checksum.getValue() != claimedSizeChecksum)
break; // entry wasn't synced correctly/fully. that's
// ok.
if (serializedSize > buffer.length) buffer = new byte[(int) (1.2 * serializedSize)];
reader.readFully(buffer, 0, serializedSize);
claimedCRC32 = reader.readLong();
} catch (EOFException eof) {
break; // last CL entry didn't get completely written. that's ok.
}
checksum.update(buffer, 0, serializedSize);
if (claimedCRC32 != checksum.getValue()) {
// this entry must not have been fsynced. probably the rest is bad too,
// but just in case there is no harm in trying them (since we still read on an entry
// boundary)
continue;
}
/* deserialize the commit log entry */
FastByteArrayInputStream bufIn = new FastByteArrayInputStream(buffer, 0, serializedSize);
RowMutation rm;
try {
// assuming version here. We've gone to lengths to make sure what gets written to the CL
// is in
// the current version. so do make sure the CL is drained prior to upgrading a node.
rm =
RowMutation.serializer()
.deserialize(
new DataInputStream(bufIn),
MessagingService.version_,
IColumnSerializer.Flag.LOCAL);
} catch (UnknownColumnFamilyException ex) {
AtomicInteger i = invalidMutations.get(ex.cfId);
if (i == null) {
i = new AtomicInteger(1);
invalidMutations.put(ex.cfId, i);
} else i.incrementAndGet();
continue;
}
if (logger.isDebugEnabled())
logger.debug(
String.format(
"replaying mutation for %s.%s: %s",
rm.getTable(),
ByteBufferUtil.bytesToHex(rm.key()),
"{" + StringUtils.join(rm.getColumnFamilies().iterator(), ", ") + "}"));
final long entryLocation = reader.getFilePointer();
final RowMutation frm = rm;
Runnable runnable =
new WrappedRunnable() {
public void runMayThrow() throws IOException {
if (Schema.instance.getKSMetaData(frm.getTable()) == null) return;
if (pointInTimeExceeded(frm)) return;
final Table table = Table.open(frm.getTable());
RowMutation newRm = new RowMutation(frm.getTable(), frm.key());
// Rebuild the row mutation, omitting column families that
// a) have already been flushed,
// b) are part of a cf that was dropped. Keep in mind that the cf.name() is suspect.
// do every thing based on the cfid instead.
for (ColumnFamily columnFamily : frm.getColumnFamilies()) {
if (Schema.instance.getCF(columnFamily.id()) == null)
// null means the cf has been dropped
continue;
ReplayPosition rp = cfPositions.get(columnFamily.id());
// replay if current segment is newer than last flushed one or,
// if it is the last known segment, if we are after the replay position
if (segment > rp.segment
|| (segment == rp.segment && entryLocation > rp.position)) {
newRm.add(columnFamily);
replayedCount.incrementAndGet();
}
}
if (!newRm.isEmpty()) {
Table.open(newRm.getTable()).apply(newRm, false);
tablesRecovered.add(table);
}
}
};
futures.add(StageManager.getStage(Stage.MUTATION).submit(runnable));
if (futures.size() > MAX_OUTSTANDING_REPLAY_COUNT) {
FBUtilities.waitOnFutures(futures);
futures.clear();
}
}
} finally {
FileUtils.closeQuietly(reader);
logger.info("Finished reading " + file);
}
}
/**
* Calculates a CRC32 checksum of the specified data.
*
* @param data - data to be hashed.
* @return - a CRC32 checksum (4 bytes).
*/
private int crc(byte[] data) {
checksum.reset();
checksum.update(data, 0, data.length);
return (int) checksum.getValue();
}
public void reset() {
summer.reset();
inSum = 0;
}