Esempio n. 1
0
 private WriteBatch enqueue(WriteCommand writeRecord) throws IOException {
   WriteBatch currentBatch = null;
   int spinnings = 0;
   int limit = 100;
   while (true) {
     if (shutdown) {
       throw new IOException("DataFileAppender Writer Thread Shutdown!");
     }
     if (firstAsyncException.get() != null) {
       throw new IOException(firstAsyncException.get());
     }
     try {
       if (batching.compareAndSet(false, true) && !shutdown) {
         if (nextWriteBatch == null) {
           DataFile file = journal.getCurrentWriteFile();
           boolean canBatch = false;
           currentBatch = new WriteBatch(file, file.getLength(), writeRecord);
           canBatch = currentBatch.canBatch(writeRecord);
           if (!canBatch) {
             file = journal.rotateWriteFile();
             currentBatch = new WriteBatch(file, file.getLength(), writeRecord);
           }
           WriteCommand controlRecord = new WriteCommand(new Location(), null, false);
           currentBatch.doFirstBatch(controlRecord, writeRecord);
           if (!writeRecord.sync) {
             inflightWrites.put(controlRecord.location, controlRecord);
             inflightWrites.put(writeRecord.location, writeRecord);
             nextWriteBatch = currentBatch;
             batching.set(false);
           } else {
             batchQueue.put(currentBatch);
             batching.set(false);
           }
           break;
         } else {
           boolean canBatch = nextWriteBatch.canBatch(writeRecord);
           if (canBatch && !writeRecord.sync) {
             nextWriteBatch.doAppendBatch(writeRecord);
             inflightWrites.put(writeRecord.location, writeRecord);
             currentBatch = nextWriteBatch;
             batching.set(false);
             break;
           } else if (canBatch && writeRecord.sync) {
             nextWriteBatch.doAppendBatch(writeRecord);
             batchQueue.put(nextWriteBatch);
             currentBatch = nextWriteBatch;
             nextWriteBatch = null;
             batching.set(false);
             break;
           } else {
             batchQueue.put(nextWriteBatch);
             nextWriteBatch = null;
             batching.set(false);
           }
         }
       } else {
         // Spin waiting for new batch ...
         if (spinnings <= limit) {
           spinnings++;
           continue;
         } else {
           Thread.sleep(250);
           continue;
         }
       }
     } catch (InterruptedException ex) {
       throw new IllegalStateException(ex.getMessage(), ex);
     }
   }
   return currentBatch;
 }
Esempio n. 2
0
  /**
   * The async processing loop that writes to the data files and does the force calls. Since the
   * file sync() call is the slowest of all the operations, this algorithm tries to 'batch' or group
   * together several file sync() requests into a single file sync() call. The batching is
   * accomplished attaching the same CountDownLatch instance to every force request in a group.
   */
  private void processQueue() {
    DataFile dataFile = null;
    RandomAccessFile file = null;
    try {
      DataByteArrayOutputStream buff =
          new DataByteArrayOutputStream(journal.getMaxWriteBatchSize());
      boolean last = false;
      while (true) {
        WriteBatch wb = batchQueue.take();

        if (shutdown) {
          last = true;
        }

        if (!wb.writes.isEmpty()) {
          boolean newOrRotated = dataFile != wb.dataFile;
          if (newOrRotated) {
            if (file != null) {
              dataFile.closeRandomAccessFile(file);
            }
            dataFile = wb.dataFile;
            file = dataFile.openRandomAccessFile();
          }

          // Write an empty batch control record.
          buff.reset();
          buff.writeInt(Journal.BATCH_CONTROL_RECORD_SIZE);
          buff.writeByte(Journal.BATCH_CONTROL_RECORD_TYPE);
          buff.write(Journal.BATCH_CONTROL_RECORD_MAGIC);
          buff.writeInt(0);
          buff.writeLong(0);

          boolean forceToDisk = false;

          WriteCommand control = wb.writes.poll();
          WriteCommand first = wb.writes.peek();
          WriteCommand latest = null;
          for (WriteCommand current : wb.writes) {
            forceToDisk |= current.sync;
            buff.writeInt(current.location.getSize());
            buff.writeByte(current.location.getType());
            buff.write(current.data.getData(), current.data.getOffset(), current.data.getLength());
            latest = current;
          }

          Buffer sequence = buff.toBuffer();

          // Now we can fill in the batch control record properly.
          buff.reset();
          buff.skip(Journal.HEADER_SIZE + Journal.BATCH_CONTROL_RECORD_MAGIC.length);
          buff.writeInt(sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
          if (journal.isChecksum()) {
            Checksum checksum = new Adler32();
            checksum.update(
                sequence.getData(),
                sequence.getOffset() + Journal.BATCH_CONTROL_RECORD_SIZE,
                sequence.getLength() - Journal.BATCH_CONTROL_RECORD_SIZE);
            buff.writeLong(checksum.getValue());
          }

          // Now do the 1 big write.
          file.seek(wb.offset);
          file.write(sequence.getData(), sequence.getOffset(), sequence.getLength());

          ReplicationTarget replicationTarget = journal.getReplicationTarget();
          if (replicationTarget != null) {
            replicationTarget.replicate(control.location, sequence, forceToDisk);
          }

          if (forceToDisk) {
            IOHelper.sync(file.getFD());
          }

          journal.setLastAppendLocation(latest.location);

          // Now that the data is on disk, remove the writes from the in
          // flight
          // cache.
          inflightWrites.remove(control.location);
          for (WriteCommand current : wb.writes) {
            if (!current.sync) {
              inflightWrites.remove(current.location);
            }
          }

          if (journal.getListener() != null) {
            try {
              journal.getListener().synced(wb.writes.toArray(new WriteCommand[wb.writes.size()]));
            } catch (Throwable ex) {
              warn(ex, ex.getMessage());
            }
          }

          // Clear unused data:
          wb.writes.clear();

          // Signal any waiting threads that the write is on disk.
          wb.latch.countDown();
        }

        if (last) {
          break;
        }
      }
    } catch (Exception e) {
      firstAsyncException.compareAndSet(null, e);
    } finally {
      try {
        if (file != null) {
          dataFile.closeRandomAccessFile(file);
        }
      } catch (Throwable ignore) {
      }
      shutdownDone.countDown();
    }
  }
Esempio n. 3
0
 DataFileAppender(Journal journal) {
   this.journal = journal;
   this.inflightWrites = journal.getInflightWrites();
 }