/**
* Displays a table of offset bits and the corresponding maximum byte offset and maximum byte
* count (aka record size) that a store may address for a given #of offset bits. This table may be
* used to choose how to parameterize the {@link WormAddressManager} and hence a {@link IRawStore}
* using that {@link WormAddressManager} so as to best leverage the 64-bit long integer as a
* persistent locator into the store.
*
* @param args unused.
*/
public static void main(final String[] args) {
final NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(true);
System.out.println("#offsetBits\tmaxOffset\tmaxByteCount");
for (int offsetBits = MIN_OFFSET_BITS; offsetBits <= MAX_OFFSET_BITS; offsetBits++) {
final WormAddressManager am = new WormAddressManager(offsetBits);
final long maxRecords = am.getMaxOffset();
final int maxRecordSize = am.getMaxByteCount();
System.out.println(
"" + offsetBits + "\t" + nf.format(maxRecords) + "\t" + nf.format(maxRecordSize));
}
}
/**
* This constructor handles the case where the file does not exist or exists but is empty
* (including files created by the temporary file creation mechanism).
*
* @param file The name of the file to be opened.
* @param bufferMode The {@link BufferMode}.
* @param useDirectBuffers true if a buffer should be allocated using {@link
* ByteBuffer#allocateDirect(int)} rather than {@link ByteBuffer#allocate(int)}. This has no
* effect for the {@link BufferMode#Disk} and {@link BufferMode#Mapped} modes.
* @param initialExtent The initial extent of the journal. The size of the journal is
* automatically increased up to the <i>maximumExtent</i> on an as necessary basis.
* @param maximumExtent The maximum extent of the journal before it will {@link
* Journal#overflow()}.
* @param create When true, the file is created if it does not exist (this is ignored for {@link
* BufferMode#Temporary} files since they are created lazily if at all).
* @param isEmptyFile This flag must be set when the temporary file mechanism is used to create a
* new temporary file otherwise an empty file is treated as an error since it does not contain
* valid root blocks.
* @param deleteOnExit When set, a <em>new</em> file will be marked for deletion when the VM
* exits. This may be used as part of a temporary store strategy.
* @param readOnly When true, the file is opened in a read-only mode and it is an error if the
* file does not exist.
* @param forceWrites When true, the file is opened in "rwd" mode and individual IOs are forced to
* disk. This option SHOULD be false since we only need to write through to disk on commit,
* not on each IO.
* @param offsetBits The #of bits out of a 64-bit long integer that are used to encode the byte
* offset as an unsigned integer. The remaining bits are used to encode the byte count (aka
* record length) as an unsigned integer. This value is <em>ignored</em> if the journal is
* being reopened, in which case the real offset bits is read from the root block of the
* journal.
* @param writeCacheEnabled When <code>true</code>, the {@link DiskOnlyStrategy} will allocate a
* direct {@link ByteBuffer} from the {@link DirectBufferPool} to service as a write cache.
* @param writeCacheBufferCount The #of buffers to allocate for the {@link WriteCacheService}.
* @param validateChecksum When <code>true</code>, the checksum stored in the root blocks of an
* existing file will be validated when the file is opened. See {@link
* Options#VALIDATE_CHECKSUM}.
* @param createTime The create time to be assigned to the root block iff a new file is created.
* @param quorumToken The current quorum token or {@link Quorum#NO_QUORUM} if the caller is not
* met with any {@link Quorum}.
* @param alternateRootBlock When <code>true</code> the prior root block will be used. This option
* may be used when a commit record is valid but the data associated with the commit point is
* invalid. There are two root blocks. Normally the one which has been most recently written
* will be loaded on restart. When this option is specified, the older of the two root blocks
* will be loaded instead. <strong>If you use this option and then do a commit then the more
* recent of the root blocks will be lost and any data associated with that commit point will
* be lost as well!</strong>
* @throws RuntimeException if there is a problem preparing the file for use by the journal.
*/
FileMetadata(
final File file,
final BufferMode bufferMode,
final boolean useDirectBuffers,
final long initialExtent,
final long maximumExtent,
final boolean create,
final boolean isEmptyFile,
boolean deleteOnExit,
final boolean readOnly,
final ForceEnum forceWrites,
final int offsetBits,
final boolean writeCacheEnabled,
final int writeCacheBufferCount,
final boolean validateChecksum,
final long createTime,
final long quorumToken,
final boolean alternateRootBlock,
final Properties properties)
throws RuntimeException {
if (file == null) throw new IllegalArgumentException();
if (bufferMode == null) throw new IllegalArgumentException();
if (bufferMode == BufferMode.Transient) {
// This mode is not a valid option in this context.
throw new IllegalArgumentException();
}
if (file.exists() && file.length() != 0)
throw new IllegalArgumentException("File exists and is not empty: " + file.getAbsolutePath());
if (readOnly && create) {
throw new IllegalArgumentException(
"'" + Options.CREATE + "' may not be used with '" + Options.READ_ONLY + "'");
}
if (readOnly && forceWrites != ForceEnum.No) {
throw new IllegalArgumentException(
"'"
+ Options.FORCE_WRITES
+ "'='"
+ forceWrites
+ "' may not be used with '"
+ Options.READ_ONLY
+ "'");
}
// check the argument. the value is only used if we are creating a new
// journal.
WormAddressManager.assertOffsetBits(offsetBits);
this.bufferMode = bufferMode;
// this.offsetBits = offsetBits;
// this.readCacheCapacity = readCacheCapacity;
//
// this.readCacheMaxRecordSize = readCacheMaxRecordSize;
this.writeCacheEnabled = writeCacheEnabled;
this.writeCacheBufferCount = writeCacheBufferCount;
this.fileMode = (readOnly ? "r" : forceWrites.asFileMode());
this.readOnly = readOnly;
this.exists = false;
this.properties = properties;
// true for a temporary file
final boolean temporary = bufferMode.equals(BufferMode.Temporary);
if (temporary) {
// override for temporary files.
deleteOnExit = true;
}
this.file = file;
if (exists && !temporary) {
if (INFO) log.info("Opening existing file: " + file.getAbsoluteFile());
} else {
if (readOnly) {
throw new RuntimeException(
"File does not exist and '"
+ Options.READ_ONLY
+ "' was specified: "
+ file.getAbsoluteFile());
}
if (!create && !isEmptyFile) {
throw new RuntimeException(
"File does not exist and '"
+ Options.CREATE
+ "' was not specified: "
+ file.getAbsoluteFile());
}
/*
* Note: a temporary file that does not exist is created _lazily_.
* See below and DiskOnlyStrategy.
*/
if (INFO)
log.info(
"Backing file: exists="
+ exists
+ ", temporary="
+ temporary
+ ", create="
+ create
+ ", readOnly="
+ readOnly
+ ", file="
+ file.getAbsoluteFile());
}
try {
/*
* Open/create the file (temporary files are not opened/created
* eagerly).
*/
// this.raf = temporary ? null : FileLockUtility.openFile(file,
// fileMode,
// bufferMode != BufferMode.Mapped);
if (!temporary) {
/*
* Open / create and obtain shared/exclusive lock if possible.
* Sets [raf] as a side-effect.
*/
opener.reopenChannel();
}
// if (exists && !temporary) {
//
// /*
// * The file already exists (but not for temporary files).
// *
// * Note: this next line will throw IOException if there is a
// * file lock contention.
// *
// * Note: [raf] was initialized by [opener.reopenChannel()]
// * above!
// */
// this.extent = raf.length();
//
// this.userExtent = extent - headerSize0;
//
// if (this.extent <= headerSize0) {
//
// /*
// * By throwing an exception for files that are not large
// * enough to contain the MAGIC, VERSION, and both root
// * blocks we avoid IO errors when trying to read those data
// * and are able to reject files based on whether they have
// * bad magic, version, or root blocks.
// */
//
// throw new RuntimeException("File too small to contain a valid journal: " +
// file.getAbsoluteFile());
//
// }
//
// // if( bufferMode != BufferMode.Disk ) {
// if (bufferMode.isFullyBuffered()) {
//
// /*
// * Verify that we can address this many bytes with this
// * strategy. The strategies that rely on an in-memory buffer
// * are all limited to the #of bytes that can be addressed by
// * an int32.
// */
//
// AbstractBufferStrategy.assertNonDiskExtent(userExtent);
//
// }
//
// /*
// * Note: The code to read the MAGIC, VERSION, and root blocks is
// * shared by DumpJournal (code is copy by value) and in part by
// * the rollback() method on AbstractJournal.
// */
//
// /*
// * Read the MAGIC and VERSION.
// */
// raf.seek(0L);
// try {
// /*
// * Note: this next line will throw IOException if there is a
// * file lock contention.
// */
// magic = raf.readInt();
// } catch (IOException ex) {
// throw new RuntimeException("Can not read magic. Is file locked by another process?",
// ex);
// }
// if (magic != MAGIC)
// throw new RuntimeException("Bad journal magic: expected=" + MAGIC + ", actual=" +
// magic);
// version = raf.readInt();
// if (version != VERSION1)
// throw new RuntimeException("Bad journal version: expected=" + VERSION1 + ", actual=" +
// version);
//
// /*
// * Check root blocks (magic, timestamps), choose root block,
// * read constants (slotSize, segmentId).
// */
// {
//
// final ChecksumUtility checker = validateChecksum ? ChecksumUtility.threadChk
// .get()
// : null;
//
// // final FileChannel channel = raf.getChannel();
// final ByteBuffer tmp0 = ByteBuffer.allocate(RootBlockView.SIZEOF_ROOT_BLOCK);
// final ByteBuffer tmp1 = ByteBuffer.allocate(RootBlockView.SIZEOF_ROOT_BLOCK);
// FileChannelUtility.readAll(opener, tmp0, OFFSET_ROOT_BLOCK0);
// FileChannelUtility.readAll(opener, tmp1, OFFSET_ROOT_BLOCK1);
// tmp0.position(0); // resets the position.
// tmp1.position(0);
// try {
// rootBlock0 = new RootBlockView(true, tmp0, checker);
// } catch (RootBlockException ex) {
// log.warn("Bad root block zero: " + ex);
// }
// try {
// rootBlock1 = new RootBlockView(false, tmp1, checker);
// } catch (RootBlockException ex) {
// log.warn("Bad root block one: " + ex);
// }
// if (rootBlock0 == null && rootBlock1 == null) {
// throw new RuntimeException(
// "Both root blocks are bad - journal is not usable: "
// + file);
// }
// if (alternateRootBlock)
// log.warn("Using alternate root block");
// /*
// * Choose the root block based on the commit counter.
// *
// * Note: The commit counters MAY be equal. This will happen
// * if we rollback the journal and override the current root
// * block with the alternate root block.
// */
// final long cc0 = rootBlock0.getCommitCounter();
// final long cc1 = rootBlock1.getCommitCounter();
// this.rootBlock = (cc0 > cc1 ? (alternateRootBlock ? rootBlock1
// : rootBlock0)
// : (alternateRootBlock ? rootBlock0 : rootBlock1));
// }
//
// // use the offset bits from the root block.
// this.offsetBits = rootBlock.getOffsetBits();
//
// /*
// * The offset into the user extent at which the next record will
// * be written.
// */
// this.nextOffset = rootBlock.getNextOffset();
//
// this.createTime = rootBlock.getCreateTime();
//
// this.closeTime = rootBlock.getCloseTime();
//
// if (closeTime != 0L && !readOnly) {
//
// throw new RuntimeException("Journal is closed for writes: closedTime=" + closeTime);
//
// }
//
// switch (bufferMode) {
// case Direct: {
// // Allocate the buffer buffer.
// buffer = (useDirectBuffers ? ByteBuffer.allocateDirect((int) userExtent) : ByteBuffer
// .allocate((int) userExtent));
// // Setup to read data from file into the buffer.
// if (nextOffset > Integer.MAX_VALUE) {
// throw new RuntimeException("This file is too large for a buffered mode: use " +
// BufferMode.Disk);
// }
// buffer.limit((int) nextOffset);
// buffer.position(0);
// if (nextOffset > 0) {
// // Read the file image into the direct buffer.
// FileChannelUtility.readAll(opener, buffer, headerSize0);
// }
// break;
// }
// case Mapped: {
// // Map the file.
// boolean loadMappedFile = false; // @todo expose as property.
// buffer = opener.reopenChannel().map(FileChannel.MapMode.READ_WRITE, headerSize0,
// extent);
// if (loadMappedFile) {
// /*
// * Load the image into mapped memory. Generally, I would
// * think that you are better off NOT loading the image.
// * When you want the image in memory, use the Direct
// * mode instead. It should be MUCH faster and has better
// * control over the amount and timing of the IO.
// */
// ((MappedByteBuffer) buffer).load();
// }
// break;
// }
// case Disk:
// buffer = null;
// break;
// case DiskWORM:
// buffer = null;
// break;
// case DiskRW:
// buffer = null;
// break;
// default:
// throw new AssertionError();
// }
//
// /*
// * Note: there should be no processing required on restart since
// * the intention of transactions that did not commit will not be
// * visible.
// */
//
// } else {
/*
* Create a new journal.
*/
if (deleteOnExit) {
// Mark the file for deletion on exit.
try {
file.deleteOnExit();
} catch (NullPointerException ex) {
/*
* Ignore NPE caused by a known Sun bug.
*
* See http://bugs.sun.com/view_bug.do?bug_id=6526376
*/
}
}
/*
* Set the initial extent.
*
* Note: since a mapped file CAN NOT be extended, we pre-extend
* it to its maximum extent here.
*/
this.extent = (bufferMode == BufferMode.Mapped ? maximumExtent : initialExtent);
this.userExtent = extent - headerSize0;
// if (bufferMode != BufferMode.Disk
// && bufferMode != BufferMode.Temporary ) {
if (userExtent > bufferMode.getMaxExtent()) {
/*
* Verify that we can address this many bytes with this
* strategy.
*/
throw new RuntimeException(AbstractBufferStrategy.ERR_MAX_EXTENT);
}
/*
* Create the root block objects (in memory).
*/
final RootBlockUtility rbu =
new RootBlockUtility(bufferMode, offsetBits, createTime, quorumToken, UUID.randomUUID());
// /*
// * The offset at which the first record will be written. This is
// * zero(0) since the buffer offset (0) is the first byte after
// * the root blocks.
// */
// nextOffset = 0; // Note: Move after we write the RBs.
magic = MAGIC;
version = CURRENT_VERSION;
if (!temporary) {
/*
* Extend the file. We do this eagerly in an attempt to
* convince the OS to place the data into a contiguous
* region on the disk.
*/
raf.setLength(extent);
/*
* Write the MAGIC and version on the file.
*/
raf.seek(0);
raf.writeInt(MAGIC);
raf.writeInt(version);
}
/*
* The root block are then written into their locations in the file.
*/
{
// final ChecksumUtility checker = ChecksumUtility.threadChk.get();
//
// // use the caller's value for offsetBits.
// this.offsetBits = offsetBits;
// final long commitCounter = 0L;
// final long firstCommitTime = 0L;
// final long lastCommitTime = 0L;
// final long commitRecordAddr = 0L;
// final long commitRecordIndexAddr = 0L;
// final UUID uuid = UUID.randomUUID(); // journal's UUID.
// final StoreTypeEnum stenum = bufferMode.getStoreType();
// if (createTime == 0L) {
// throw new IllegalArgumentException(
// "Create time may not be zero.");
// }
// this.createTime = createTime;
// this.closeTime = 0L;
// final long blockSequence = IRootBlockView.NO_BLOCK_SEQUENCE;
// final IRootBlockView rootBlock0 = new RootBlockView(true,
// offsetBits, nextOffset, firstCommitTime,
// lastCommitTime, commitCounter, commitRecordAddr,
// commitRecordIndexAddr, uuid, //
// blockSequence, quorumToken,//
// 0L, 0L, stenum, createTime, closeTime, RootBlockView.currentVersion, checker);
// final IRootBlockView rootBlock1 = new RootBlockView(false,
// offsetBits, nextOffset, firstCommitTime,
// lastCommitTime, commitCounter, commitRecordAddr,
// commitRecordIndexAddr, uuid, //
// blockSequence, quorumToken,//
// 0L, 0L, stenum, createTime, closeTime, RootBlockView.currentVersion, checker);
// take various values from the current RB.
this.nextOffset = rbu.rootBlock.getNextOffset();
this.offsetBits = rbu.rootBlock.getOffsetBits();
this.createTime = rbu.rootBlock.getCreateTime();
this.closeTime = rbu.rootBlock.getCloseTime();
final IRootBlockView rootBlock0 = rbu.rootBlock0;
final IRootBlockView rootBlock1 = rbu.rootBlock1;
if (!temporary) {
// FileChannel channel = raf.getChannel();
FileChannelUtility.writeAll(opener, rootBlock0.asReadOnlyBuffer(), OFFSET_ROOT_BLOCK0);
FileChannelUtility.writeAll(opener, rootBlock1.asReadOnlyBuffer(), OFFSET_ROOT_BLOCK1);
/*
* Force the changes to disk. We also force the file
* metadata to disk since we just changed the file size and
* we do not want to loose track of that.
*/
opener.reopenChannel().force(true);
}
this.rootBlock = rootBlock0;
}
switch (bufferMode) {
case Direct:
/*
* Allocate the buffer.
*
* Note that we do not read in any data since no user data
* has been written and the root blocks are not cached in
* the buffer to avoid possible overwrites.
*/
buffer =
(useDirectBuffers
? ByteBuffer.allocateDirect((int) userExtent)
: ByteBuffer.allocate((int) userExtent));
break;
case Mapped:
/*
* Map the file starting from the first byte of the user
* space and continuing through the entire user extent.
*/
if (INFO) log.info("Mapping file=" + file);
buffer =
opener.reopenChannel().map(FileChannel.MapMode.READ_WRITE, headerSize0, userExtent);
break;
case TemporaryRW:
case DiskRW:
buffer = null;
break;
case Disk:
buffer = null;
break;
case DiskWORM:
buffer = null;
break;
case Temporary:
buffer = null;
break;
case MemStore:
buffer = null;
break;
default:
throw new AssertionError();
}
// }
this.useChecksums = useChecksums(rootBlock);
} catch (IOException ex) {
throw new RuntimeException("file=" + file, ex);
}
}
