private WriteBatch enqueue(WriteCommand writeRecord) throws IOException {
WriteBatch currentBatch = null;
int spinnings = 0;
int limit = 100;
while (true) {
if (shutdown) {
throw new IOException("DataFileAppender Writer Thread Shutdown!");
}
if (firstAsyncException.get() != null) {
throw new IOException(firstAsyncException.get());
}
try {
if (batching.compareAndSet(false, true) && !shutdown) {
if (nextWriteBatch == null) {
DataFile file = journal.getCurrentWriteFile();
boolean canBatch = false;
currentBatch = new WriteBatch(file, file.getLength(), writeRecord);
canBatch = currentBatch.canBatch(writeRecord);
if (!canBatch) {
file = journal.rotateWriteFile();
currentBatch = new WriteBatch(file, file.getLength(), writeRecord);
}
WriteCommand controlRecord = new WriteCommand(new Location(), null, false);
currentBatch.doFirstBatch(controlRecord, writeRecord);
if (!writeRecord.sync) {
inflightWrites.put(controlRecord.location, controlRecord);
inflightWrites.put(writeRecord.location, writeRecord);
nextWriteBatch = currentBatch;
batching.set(false);
} else {
batchQueue.put(currentBatch);
batching.set(false);
}
break;
} else {
boolean canBatch = nextWriteBatch.canBatch(writeRecord);
if (canBatch && !writeRecord.sync) {
nextWriteBatch.doAppendBatch(writeRecord);
inflightWrites.put(writeRecord.location, writeRecord);
currentBatch = nextWriteBatch;
batching.set(false);
break;
} else if (canBatch && writeRecord.sync) {
nextWriteBatch.doAppendBatch(writeRecord);
batchQueue.put(nextWriteBatch);
currentBatch = nextWriteBatch;
nextWriteBatch = null;
batching.set(false);
break;
} else {
batchQueue.put(nextWriteBatch);
nextWriteBatch = null;
batching.set(false);
}
}
} else {
// Spin waiting for new batch ...
if (spinnings <= limit) {
spinnings++;
continue;
} else {
Thread.sleep(250);
continue;
}
}
} catch (InterruptedException ex) {
throw new IllegalStateException(ex.getMessage(), ex);
}
}
return currentBatch;
}
/**
* The async processing loop that writes to the data files and does the force calls. Since the
* file sync() call is the slowest of all the operations, this algorithm tries to 'batch' or group
* together several file sync() requests into a single file sync() call. The batching is
* accomplished attaching the same CountDownLatch instance to every force request in a group.
*/
private void processQueue() {
DataFile dataFile = null;
RandomAccessFile file = null;
try {
DataByteArrayOutputStream buff =
new DataByteArrayOutputStream(journal.getMaxWriteBatchSize());
boolean last = false;
while (true) {
WriteBatch wb = batchQueue.take();
if (shutdown) {
last = true;
}
if (!wb.writes.isEmpty()) {
boolean newOrRotated = dataFile != wb.dataFile;
if (newOrRotated) {
if (file != null) {
dataFile.closeRandomAccessFile(file);
}
dataFile = wb.dataFile;
file = dataFile.openRandomAccessFile();
}
// Write an empty batch control record.
buff.reset();
buff.writeInt(Journal.BATCH_CONTROL_RECORD_SIZE);
buff.writeByte(Journal.BATCH_CONTROL_RECORD_TYPE);
buff.write(Journal.BATCH_CONTROL_RECORD_MAGIC);
buff.writeInt(0);
buff.writeLong(0);
boolean forceToDisk = false;
WriteCommand control = wb.writes.poll();
WriteCommand first = wb.writes.peek();
WriteCommand latest = null;
for (WriteCommand current : wb.writes) {
forceToDisk |= current.sync;
buff.writeInt(current.location.getSize());
buff.writeByte(current.location.getType());
buff.write(current.data.getData(), current.data.getOffset(), current.data.getLength());
latest = current;
}
Buffer sequence = buff.toBuffer();
// Now we can fill in the batch control record properly.
buff.reset();
buff.skip(Journal.HEADER_SIZE + Journal.BATCH_CONTROL_RECORD_MAGIC.length);
buff.writeInt(sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
if (journal.isChecksum()) {
Checksum checksum = new Adler32();
checksum.update(
sequence.getData(),
sequence.getOffset() + Journal.BATCH_CONTROL_RECORD_SIZE,
sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
buff.writeLong(checksum.getValue());
}
// Now do the 1 big write.
file.seek(wb.offset);
file.write(sequence.getData(), sequence.getOffset(), sequence.getLength());
ReplicationTarget replicationTarget = journal.getReplicationTarget();
if (replicationTarget != null) {
replicationTarget.replicate(control.location, sequence, forceToDisk);
}
if (forceToDisk) {
IOHelper.sync(file.getFD());
}
journal.setLastAppendLocation(latest.location);
// Now that the data is on disk, remove the writes from the in
// flight
// cache.
inflightWrites.remove(control.location);
for (WriteCommand current : wb.writes) {
if (!current.sync) {
inflightWrites.remove(current.location);
}
}
if (journal.getListener() != null) {
try {
journal.getListener().synced(wb.writes.toArray(new WriteCommand[wb.writes.size()]));
} catch (Throwable ex) {
warn(ex, ex.getMessage());
}
}
// Clear unused data:
wb.writes.clear();
// Signal any waiting threads that the write is on disk.
wb.latch.countDown();
}
if (last) {
break;
}
}
} catch (Exception e) {
firstAsyncException.compareAndSet(null, e);
} finally {
try {
if (file != null) {
dataFile.closeRandomAccessFile(file);
}
} catch (Throwable ignore) {
}
shutdownDone.countDown();
}
}
DataFileAppender(Journal journal) {
this.journal = journal;
this.inflightWrites = journal.getInflightWrites();
}