private void insertOrUpdate(long recid, DataIO.DataOutputByteArray out, boolean isInsert) {
if (CC.ASSERT) assertWriteLocked(lockPos(recid));
// TODO assert indexTable state, record should already exist/not exist
final int realSize = out == null ? 0 : out.pos;
final int shiftedSize = out == null ? 0 : realSize + 1; // one additional state to indicate null
final int headSize =
1
+ // instruction
DataIO.packLongSize(longParitySet(recid))
+ // recid
DataIO.packLongSize(longParitySet(shiftedSize)); // length
long offset = alloc(headSize, headSize + realSize);
final long origOffset = offset;
// ensure available worst case scenario
vol.ensureAvailable(offset + headSize + realSize);
// instruction
vol.putUnsignedByte(offset, isInsert ? I_INSERT : I_UPDATE);
offset++;
// recid
offset += vol.putPackedLong(offset, longParitySet(recid));
// size
offset += vol.putPackedLong(offset, longParitySet(shiftedSize));
if (realSize != 0) vol.putDataOverlap(offset, out.buf, 0, out.pos);
// -3 is null record
// -2 is zero size record
indexTablePut(recid, out == null ? -3 : (realSize == 0) ? -2 : origOffset);
}
@Test
public void testSixLong() {
byte[] b = new byte[8];
for (long i = 0; i >>> 48 == 0; i = i + 1 + i / 10000) {
DataIO.putSixLong(b, 2, i);
assertEquals(i, DataIO.getSixLong(b, 2));
}
}
@Test
public void testPackLongSize() {
assertEquals(
"packLongSize should have returned 1 since number 1 can be represented using 1 byte when packed",
1,
DataIO.packLongSize(1));
assertEquals(
"packLongSize should have returned 2 since 1 << 7 can be represented using 2 bytes when packed",
2,
DataIO.packLongSize(1 << 7));
assertEquals(
"packLongSize should have returned 10 since 1 << 63 can be represented using 10 bytes when packed",
10,
DataIO.packLongSize(1 << 63));
}
@Test
public void testPackLong_WithStreams() throws IOException {
for (long valueToPack = 0;
valueToPack < Long.MAX_VALUE && valueToPack >= 0;
valueToPack = random.nextInt(2) + valueToPack * 2) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
DataIO.packLong(outputStream, valueToPack);
DataIO.packLong(outputStream, -valueToPack);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
long unpackedLong = DataIO.unpackLong(inputStream);
assertEquals("Packed and unpacked values do not match", valueToPack, unpackedLong);
unpackedLong = DataIO.unpackLong(inputStream);
assertEquals("Packed and unpacked values do not match", -valueToPack, unpackedLong);
}
}
@Test(expected = EOFException.class)
public void testReadFully_throws_exception_if_not_enough_data() throws IOException {
InputStream inputStream = new ByteArrayInputStream(new byte[0]);
DataIO.readFully(inputStream, new byte[1]);
fail(
"An EOFException should have occurred by now since there are not enough bytes to read from the InputStream");
}
@Test(expected = EOFException.class)
public void testUnpackLong_withInputStream_throws_exception_when_stream_is_empty()
throws IOException {
DataIO.unpackLong(new ByteArrayInputStream(new byte[0]));
fail(
"An EOFException should have occurred by now since there are no bytes to read from the InputStream");
}
@Test
public void packInt() throws IOException {
DataInputByteArray in = new DataInputByteArray(new byte[20]);
DataOutputByteArray out = new DataOutputByteArray();
out.buf = in.buf;
for (int i = 0; i > 0; i = i + 1 + i / 10000) {
in.pos = 10;
out.pos = 10;
DataIO.packInt((DataOutput) out, i);
long i2 = DataIO.unpackInt(in);
assertEquals(i, i2);
assertEquals(in.pos, out.pos);
}
}
@Override
protected <A> A get2(long recid, Serializer<A> serializer) {
if (CC.ASSERT) assertReadLocked(recid);
long offset = modified[lockPos(recid)].get(recid);
if (offset == 0) {
try {
offset = indexTable.getLong(recid * 8);
} catch (ArrayIndexOutOfBoundsException e) {
// TODO this code should be aware if indexTable internals?
throw new DBException.EngineGetVoid();
}
}
if (offset == -3 || offset == -1) // null, preallocated or deleted
return null;
if (offset == 0) { // non existent
throw new DBException.EngineGetVoid();
}
if (offset == -2) {
// zero size record
return deserialize(serializer, 0, new DataIO.DataInputByteArray(new byte[0]));
}
final long packedRecidSize = DataIO.packLongSize(longParitySet(recid));
if (CC.ASSERT) {
int instruction = vol.getUnsignedByte(offset);
if (instruction != I_UPDATE && instruction != I_INSERT)
throw new DBException.DataCorruption("wrong instruction " + instruction);
long recid2 = vol.getPackedLong(offset + 1);
if (packedRecidSize != recid2 >>> 60)
throw new DBException.DataCorruption("inconsistent recid len");
recid2 = longParityGet(recid2 & DataIO.PACK_LONG_RESULT_MASK);
if (recid != recid2) throw new DBException.DataCorruption("recid does not match");
}
offset +=
1
+ // instruction size
packedRecidSize; // recid size
// read size
long size = vol.getPackedLong(offset);
offset += size >>> 60;
size = longParityGet(size & DataIO.PACK_LONG_RESULT_MASK);
size -= 1; // normalize size
if (CC.ASSERT && size <= 0) throw new DBException.DataCorruption("wrong size");
DataInput input = vol.getDataInputOverlap(offset, (int) size);
return deserialize(serializer, (int) size, input);
}
protected long indexHeaderChecksum() {
long ret = 0;
for (long offset = 0; offset < IO_USER_START; offset += 8) {
if (offset == IO_INDEX_SUM) continue;
long indexVal = index.getLong(offset);
ret += indexVal + DataIO.longHash(indexVal + offset);
}
return ret;
}
@Test
public void testFillLowBits() {
for (int bitCount = 0; bitCount < 64; bitCount++) {
assertEquals(
"fillLowBits should return a long value with 'bitCount' least significant bits set to one",
(1L << bitCount) - 1,
DataIO.fillLowBits(bitCount));
}
}
@Test
public void commitChecksum() {
WriteAheadLog wal = new WriteAheadLog(null);
wal.open(WriteAheadLog.NOREPLAY);
wal.startNextFile();
wal.walPutLong(111L, 1000);
wal.commit();
long offset1 = wal.fileOffset - 5;
int checksum1 = DataIO.longHash(wal.curVol.hash(16, offset1 - 16, 111L));
assertEquals(checksum1, wal.curVol.getInt(offset1 + 1));
wal.walPutLong(111L, 1000);
wal.commit();
long offset2 = wal.fileOffset - 5;
int checksum2 =
checksum1 + DataIO.longHash(wal.curVol.hash(offset1 + 5, offset2 - offset1 - 5, 111L));
assertEquals(checksum2, wal.curVol.getInt(offset2 + 1));
}
@Test
public void testPutLong() throws IOException {
for (long valueToPut = 0;
valueToPut < Long.MAX_VALUE && valueToPut >= 0;
valueToPut = random.nextInt(2) + valueToPut * 2) {
byte[] buffer = new byte[20];
DataIO.putLong(buffer, 2, valueToPut);
long returned = DataIO.getLong(buffer, 2);
assertEquals(
"The value that was put and the value returned from getLong do not match",
valueToPut,
returned);
DataIO.putLong(buffer, 2, -valueToPut);
returned = DataIO.getLong(buffer, 2);
assertEquals(
"The value that was put and the value returned from getLong do not match",
-valueToPut,
returned);
}
}
@Override
protected void putDataSingleWithLink(
int segment, long offset, long link, byte[] buf, int bufPos, int size) {
if (CC.ASSERT && (size & 0xFFFF) != size) throw new DBException.DataCorruption();
// TODO optimize so array copy is not necessary, that means to clone and modify
// putDataSingleWithoutLink method
byte[] buf2 = new byte[size + 8];
DataIO.putLong(buf2, 0, link);
System.arraycopy(buf, bufPos, buf2, 8, size);
putDataSingleWithoutLink(segment, offset, buf2, 0, buf2.length);
}
@Test
public void testReadFully_with_data_length_same_as_buffer_length() throws IOException {
byte[] inputBuffer = new byte[] {1, 2, 3, 4};
InputStream in = new ByteArrayInputStream(inputBuffer);
byte[] outputBuffer = new byte[4];
DataIO.readFully(in, outputBuffer);
assertArrayEquals(
"The passed buffer should be filled with the whole content of the InputStream"
+ " since the buffer length is exactly same as the data length",
inputBuffer,
outputBuffer);
}
@Test
public void testReadFully_with_too_much_data() throws IOException {
byte[] inputBuffer = new byte[] {1, 2, 3, 4};
InputStream in = new ByteArrayInputStream(inputBuffer);
byte[] outputBuffer = new byte[3];
DataIO.readFully(in, outputBuffer);
byte[] expected = new byte[] {1, 2, 3};
assertArrayEquals(
"The passed buffer should be filled with the first three bytes read from the InputStream",
expected,
outputBuffer);
}
@Override
protected <A> void delete2(long recid, Serializer<A> serializer) {
if (CC.ASSERT) assertWriteLocked(lockPos(recid));
final int headSize = 1 + DataIO.packLongSize(longParitySet(recid));
long offset = alloc(headSize, headSize);
vol.ensureAvailable(offset + headSize);
vol.putUnsignedByte(offset, I_DELETE); // delete instruction
offset++;
vol.putPackedLong(offset, longParitySet(recid));
indexTablePut(recid, -1); // -1 is deleted record
}
@Override
public long preallocate() {
long recid = highestRecid.incrementAndGet();
Lock lock = locks[lockPos(recid)].writeLock();
lock.lock();
try {
final int headSize = 1 + DataIO.packLongSize(longParitySet(recid));
long offset = alloc(headSize, headSize);
vol.ensureAvailable(offset + headSize);
vol.putUnsignedByte(offset, I_PREALLOC);
offset++;
vol.putPackedLong(offset, longParitySet(recid));
indexTablePut(recid, -3);
} finally {
lock.unlock();
}
return recid;
}
@Test
public void testNextPowTwoLong() {
assertEquals(1, DataIO.nextPowTwo(1L));
assertEquals(2, DataIO.nextPowTwo(2L));
assertEquals(4, DataIO.nextPowTwo(3L));
assertEquals(4, DataIO.nextPowTwo(4L));
assertEquals(64, DataIO.nextPowTwo(33L));
assertEquals(64, DataIO.nextPowTwo(61L));
assertEquals(1024, DataIO.nextPowTwo(777L));
assertEquals(1024, DataIO.nextPowTwo(1024L));
assertEquals(1073741824, DataIO.nextPowTwo(1073741824L - 100));
assertEquals(1073741824, DataIO.nextPowTwo((long) (1073741824 * 0.7)));
assertEquals(1073741824, DataIO.nextPowTwo(1073741824L));
}
@Test
public void testHexaConversion() {
byte[] b = new byte[] {11, 112, 11, 0, 39, 90};
assertTrue(Serializer.BYTE_ARRAY.equals(b, DataIO.fromHexa(DataIO.toHexa(b))));
}
@Test
public void testNextPowTwo() {
assertEquals(1, DataIO.nextPowTwo(1));
assertEquals(2, DataIO.nextPowTwo(2));
assertEquals(4, DataIO.nextPowTwo(3));
assertEquals(4, DataIO.nextPowTwo(4));
assertEquals(64, DataIO.nextPowTwo(33));
assertEquals(64, DataIO.nextPowTwo(61));
assertEquals(1024, DataIO.nextPowTwo(777));
assertEquals(1024, DataIO.nextPowTwo(1024));
assertEquals(1073741824, DataIO.nextPowTwo(1073741824 - 100));
assertEquals(1073741824, DataIO.nextPowTwo((int) (1073741824 * 0.7)));
assertEquals(1073741824, DataIO.nextPowTwo(1073741824));
}
/**
* Tests if the specified object is a key in this table.
*
* @param key possible key
* @return <tt>true</tt> if and only if the specified object is a key in this table, as determined
* by the <tt>equals</tt> method; <tt>false</tt> otherwise.
* @throws NullPointerException if the specified key is null
*/
public boolean containsKey(long key) {
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).containsKey(key, hash);
}
/**
* Returns the value to which the specified key is mapped, or {@code null} if this map contains no
* mapping for the key.
*
* <p>More formally, if this map contains a mapping from a key {@code k} to a value {@code keys}
* such that {@code key.equals(k)}, then this method returns {@code keys}; otherwise it returns
* {@code null}. (There can be at most one such mapping.)
*
* @throws NullPointerException if the specified key is null
*/
@Override
public V get(long key) {
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).get(key, hash);
}
/**
* @return the previous value associated with the specified key, or <tt>null</tt> if there was no
* mapping for the key
* @throws NullPointerException if the specified key or value is null
*/
public V replace(long key, V value) {
if (value == null) throw new NullPointerException();
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).replace(key, hash, value);
}
/** @throws NullPointerException if any of the arguments are null */
public boolean replace(long key, V oldValue, V newValue) {
if (oldValue == null || newValue == null) throw new NullPointerException();
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).replace(key, hash, oldValue, newValue);
}
/** @throws NullPointerException if the specified key is null */
public boolean remove(long key, Object value) {
final int hash = DataIO.longHash(key ^ hashSalt);
return value != null && segmentFor(hash).remove(key, hash, value) != null;
}
/**
* Removes the key (and its corresponding value) from this map. This method does nothing if the
* key is not in the map.
*
* @param key the key that needs to be removed
* @return the previous value associated with <tt>key</tt>, or <tt>null</tt> if there was no
* mapping for <tt>key</tt>
* @throws NullPointerException if the specified key is null
*/
@Override
public V remove(long key) {
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).remove(key, hash, null);
}
/**
* @return the previous value associated with the specified key, or <tt>null</tt> if there was no
* mapping for the key
* @throws NullPointerException if the specified key or value is null
*/
public V putIfAbsent(long key, V value) {
if (value == null) throw new NullPointerException();
final int hash = DataIO.longHash(key ^ hashSalt);
return segmentFor(hash).put(key, hash, value, true);
}
