Location storeItem(Buffer data, byte type, boolean sync) throws IOException {
// Write the packet into our internal buffer.
int size = Journal.HEADER_SIZE + data.getLength();
Location location = new Location();
location.setSize(size);
location.setType(type);
WriteCommand write = new WriteCommand(location, data, sync);
WriteBatch batch = enqueue(write);
location.setLatch(batch.latch);
if (sync) {
try {
batch.latch.await();
} catch (InterruptedException e) {
throw new InterruptedIOException();
}
}
return location;
}
/**
* The async processing loop that writes to the data files and does the force calls. Since the
* file sync() call is the slowest of all the operations, this algorithm tries to 'batch' or group
* together several file sync() requests into a single file sync() call. The batching is
* accomplished attaching the same CountDownLatch instance to every force request in a group.
*/
private void processQueue() {
DataFile dataFile = null;
RandomAccessFile file = null;
try {
DataByteArrayOutputStream buff =
new DataByteArrayOutputStream(journal.getMaxWriteBatchSize());
boolean last = false;
while (true) {
WriteBatch wb = batchQueue.take();
if (shutdown) {
last = true;
}
if (!wb.writes.isEmpty()) {
boolean newOrRotated = dataFile != wb.dataFile;
if (newOrRotated) {
if (file != null) {
dataFile.closeRandomAccessFile(file);
}
dataFile = wb.dataFile;
file = dataFile.openRandomAccessFile();
}
// Write an empty batch control record.
buff.reset();
buff.writeInt(Journal.BATCH_CONTROL_RECORD_SIZE);
buff.writeByte(Journal.BATCH_CONTROL_RECORD_TYPE);
buff.write(Journal.BATCH_CONTROL_RECORD_MAGIC);
buff.writeInt(0);
buff.writeLong(0);
boolean forceToDisk = false;
WriteCommand control = wb.writes.poll();
WriteCommand first = wb.writes.peek();
WriteCommand latest = null;
for (WriteCommand current : wb.writes) {
forceToDisk |= current.sync;
buff.writeInt(current.location.getSize());
buff.writeByte(current.location.getType());
buff.write(current.data.getData(), current.data.getOffset(), current.data.getLength());
latest = current;
}
Buffer sequence = buff.toBuffer();
// Now we can fill in the batch control record properly.
buff.reset();
buff.skip(Journal.HEADER_SIZE + Journal.BATCH_CONTROL_RECORD_MAGIC.length);
buff.writeInt(sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
if (journal.isChecksum()) {
Checksum checksum = new Adler32();
checksum.update(
sequence.getData(),
sequence.getOffset() + Journal.BATCH_CONTROL_RECORD_SIZE,
sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
buff.writeLong(checksum.getValue());
}
// Now do the 1 big write.
file.seek(wb.offset);
file.write(sequence.getData(), sequence.getOffset(), sequence.getLength());
ReplicationTarget replicationTarget = journal.getReplicationTarget();
if (replicationTarget != null) {
replicationTarget.replicate(control.location, sequence, forceToDisk);
}
if (forceToDisk) {
IOHelper.sync(file.getFD());
}
journal.setLastAppendLocation(latest.location);
// Now that the data is on disk, remove the writes from the in
// flight
// cache.
inflightWrites.remove(control.location);
for (WriteCommand current : wb.writes) {
if (!current.sync) {
inflightWrites.remove(current.location);
}
}
if (journal.getListener() != null) {
try {
journal.getListener().synced(wb.writes.toArray(new WriteCommand[wb.writes.size()]));
} catch (Throwable ex) {
warn(ex, ex.getMessage());
}
}
// Clear unused data:
wb.writes.clear();
// Signal any waiting threads that the write is on disk.
wb.latch.countDown();
}
if (last) {
break;
}
}
} catch (Exception e) {
firstAsyncException.compareAndSet(null, e);
} finally {
try {
if (file != null) {
dataFile.closeRandomAccessFile(file);
}
} catch (Throwable ignore) {
}
shutdownDone.countDown();
}
}