private void writeAllRows() {
if (file == null) {
Database db = session.getDatabase();
String fileName = db.createTempFile();
file = db.openFile(fileName, "rw", false);
file.setCheckedWriting(false);
file.seek(FileStore.HEADER_LENGTH);
rowBuff = Data.create(db, Constants.DEFAULT_PAGE_SIZE);
file.seek(FileStore.HEADER_LENGTH);
}
Data buff = rowBuff;
initBuffer(buff);
for (int i = 0, size = list.size(); i < size; i++) {
if (i > 0 && buff.length() > Constants.IO_BUFFER_SIZE) {
flushBuffer(buff);
initBuffer(buff);
}
Row r = list.get(i);
writeRow(buff, r);
}
flushBuffer(buff);
file.autoDelete();
list.clear();
memory = 0;
}
/**
* Get the row at the given index.
*
* @param at the index
* @return the row
*/
Row getRowAt(int at) {
Row r = rows[at];
if (r == null) {
if (firstOverflowPageId == 0) {
r = readRow(data, offsets[at], columnCount);
} else {
if (rowRef != null) {
r = rowRef.get();
if (r != null) {
return r;
}
}
PageStore store = index.getPageStore();
Data buff = store.createData();
int pageSize = store.getPageSize();
int offset = offsets[at];
buff.write(data.getBytes(), offset, pageSize - offset);
int next = firstOverflowPageId;
do {
PageDataOverflow page = index.getPageOverflow(next);
next = page.readInto(buff);
} while (next != 0);
overflowRowSize = pageSize + buff.length();
r = readRow(buff, 0, columnCount);
}
r.setKey(keys[at]);
if (firstOverflowPageId != 0) {
rowRef = new SoftReference<Row>(r);
} else {
rows[at] = r;
memoryChange(true, r);
}
}
return r;
}
/** Read the page from the disk. */
private void read() {
data.reset();
data.readByte();
data.readShortInt();
trunk = data.readInt();
logKey = data.readInt();
}
/**
* Prepare a transaction.
*
* @param session the session
* @param transaction the name of the transaction
*/
void prepareCommit(Session session, String transaction) {
if (trace.isDebugEnabled()) {
trace.debug("log prepare commit s: " + session.getId() + ", " + transaction);
}
if (store.getDatabase().getPageStore() == null) {
// database already closed
return;
}
// store it on a separate log page
int pageSize = store.getPageSize();
pageOut.flush();
pageOut.fillPage();
Data buffer = getBuffer();
buffer.writeByte((byte) PREPARE_COMMIT);
buffer.writeVarInt(session.getId());
buffer.writeString(transaction);
if (buffer.length() >= PageStreamData.getCapacity(pageSize)) {
throw DbException.getInvalidValueException("transaction name (too long)", transaction);
}
write(buffer);
// store it on a separate log page
flushOut();
pageOut.fillPage();
if (store.getDatabase().getFlushOnEachCommit()) {
flush();
}
}
/** Write the header data. */
void initWrite() {
data = store.createData();
data.writeByte((byte) Page.TYPE_STREAM_DATA);
data.writeShortInt(0);
data.writeInt(trunk);
data.writeInt(logKey);
remaining = store.getPageSize() - data.length();
}
/**
* Read a row from the data page at the given position.
*
* @param data the data page
* @param pos the position to read from
* @param columnCount the number of columns
* @return the row
*/
private static Row readRow(Data data, int pos, int columnCount) {
Value[] values = new Value[columnCount];
synchronized (data) {
data.setPos(pos);
for (int i = 0; i < columnCount; i++) {
values[i] = data.readValue();
}
}
return RegularTable.createRow(values);
}
@Override
public void write() {
data = store.createData();
data.writeByte((byte) Page.TYPE_FREE_LIST);
data.writeShortInt(0);
for (int i = 0; i < pageCount; i += 8) {
data.writeByte((byte) used.getByte(i));
}
store.writePage(getPos(), data);
}
/** Read the page from the disk. */
private void read() {
data.reset();
data.readByte();
data.readShortInt();
for (int i = 0; i < pageCount; i += 8) {
int x = data.readByte() & 255;
used.setByte(i, x);
}
full = false;
}
/** Switch to a new log section. */
void checkpoint() {
Data buffer = getBuffer();
buffer.writeByte((byte) CHECKPOINT);
write(buffer);
undo = new BitField();
logSectionId++;
logPos = 0;
pageOut.flush();
pageOut.fillPage();
int currentDataPage = pageOut.getCurrentDataPageId();
logSectionPageMap.put(logSectionId, currentDataPage);
}
/**
* Set the state of an in-doubt transaction.
*
* @param sessionId the session
* @param pageId the page where the commit was prepared
* @param commit whether the transaction should be committed
*/
void setInDoubtTransactionState(int sessionId, int pageId, boolean commit) {
PageStreamData d = (PageStreamData) store.getPage(pageId);
d.initWrite();
Data buff = store.createData();
buff.writeByte((byte) (commit ? COMMIT : ROLLBACK));
buff.writeVarInt(sessionId);
byte[] bytes = buff.getBytes();
d.write(bytes, 0, bytes.length);
bytes = new byte[d.getRemaining()];
d.write(bytes, 0, bytes.length);
d.write();
}
/**
* A table is truncated.
*
* @param session the session
* @param tableId the table id
*/
void logTruncate(Session session, int tableId) {
if (trace.isDebugEnabled()) {
trace.debug("log truncate s: " + session.getId() + " table: " + tableId);
}
session.addLogPos(logSectionId, logPos);
logPos++;
Data buffer = getBuffer();
buffer.writeByte((byte) TRUNCATE);
buffer.writeVarInt(session.getId());
buffer.writeVarInt(tableId);
write(buffer);
}
private void freeLogPages(IntArray pages) {
if (trace.isDebugEnabled()) {
trace.debug("log frees " + pages.get(0) + ".." + pages.get(pages.size() - 1));
}
Data buffer = getBuffer();
buffer.writeByte((byte) FREE_LOG);
int size = pages.size();
buffer.writeVarInt(size);
for (int i = 0; i < size; i++) {
buffer.writeVarInt(pages.get(i));
}
write(buffer);
}
/**
* Copy a String from a reader to an output stream.
*
* @param source the reader
* @param target the output stream
*/
public static void copyString(Reader source, OutputStream target) throws IOException {
char[] buff = new char[Constants.IO_BUFFER_SIZE];
Data d = new Data(null, new byte[3 * Constants.IO_BUFFER_SIZE]);
while (true) {
int l = source.read(buff);
if (l < 0) {
break;
}
d.writeStringWithoutLength(buff, l);
target.write(d.data, 0, d.pos);
d.reset();
}
}
/**
* Read a row from an input stream.
*
* @param in the input stream
* @param data a temporary buffer
* @return the row
*/
public static Row readRow(DataReader in, Data data) throws IOException {
long key = in.readVarLong();
int len = in.readVarInt();
data.reset();
data.checkCapacity(len);
in.readFully(data.getBytes(), len);
int columnCount = data.readVarInt();
Value[] values = new Value[columnCount];
for (int i = 0; i < columnCount; i++) {
values[i] = data.readValue();
}
Row row = new Row(values, Row.MEMORY_CALCULATE);
row.setKey(key);
return row;
}
/**
* Get the number of bytes required for the data.
*
* @param dummy the template buffer
* @return the number of bytes
*/
public int getByteCount(Data dummy) {
int size = 0;
for (Value v : data) {
size += dummy.getValueLen(v);
}
return size;
}
/**
* Mark a transaction as committed.
*
* @param sessionId the session
*/
void commit(int sessionId) {
if (trace.isDebugEnabled()) {
trace.debug("log commit s: " + sessionId);
}
if (store.getDatabase().getPageStore() == null) {
// database already closed
return;
}
Data buffer = getBuffer();
buffer.writeByte((byte) COMMIT);
buffer.writeVarInt(sessionId);
write(buffer);
if (store.getDatabase().getFlushOnEachCommit()) {
flush();
}
}
private ResultDiskBuffer(ResultDiskBuffer parent) {
this.parent = parent;
rowBuff = Data.create(parent.rowBuff.getHandler(), Constants.DEFAULT_PAGE_SIZE);
file = parent.file;
if (parent.tapes != null) {
tapes = New.arrayList();
for (ResultDiskTape t : parent.tapes) {
ResultDiskTape t2 = new ResultDiskTape();
t2.pos = t2.start = t.start;
t2.end = t.end;
tapes.add(t2);
}
} else {
tapes = null;
}
if (parent.mainTape != null) {
mainTape = new ResultDiskTape();
mainTape.pos = FileStore.HEADER_LENGTH;
mainTape.start = parent.mainTape.start;
mainTape.end = parent.mainTape.end;
} else {
mainTape = null;
}
sort = parent.sort;
columnCount = parent.columnCount;
maxBufferSize = parent.maxBufferSize;
}
/**
* Change the parent page id.
*
* @param id the new parent page id
*/
void setParentPageId(int id) {
index.getPageStore().logUndo(this, data);
parentPageId = id;
if (written) {
changeCount = index.getPageStore().getChangeCount();
data.setInt(START_PARENT, parentPageId);
}
}
private void flushBuffer(Data buff) {
buff.checkCapacity(1);
buff.writeByte((byte) 0);
buff.fillAligned();
buff.setInt(0, buff.length() / Constants.FILE_BLOCK_SIZE);
file.write(buff.getBytes(), 0, buff.length());
}
public String toString() {
return "["
+ getPos()
+ "] stream data key:"
+ logKey
+ " pos:"
+ data.length()
+ " remaining:"
+ remaining;
}
/**
* Get the next row from the list.
*
* @return the next row
*/
public Row next() {
Row r;
if (file == null) {
r = list.get(index++);
} else {
if (listIndex >= list.size()) {
list.clear();
listIndex = 0;
Data buff = rowBuff;
buff.reset();
int min = Constants.FILE_BLOCK_SIZE;
file.readFully(buff.getBytes(), 0, min);
int len = buff.readInt() * Constants.FILE_BLOCK_SIZE;
buff.checkCapacity(len);
if (len - min > 0) {
file.readFully(buff.getBytes(), min, len - min);
}
while (true) {
r = readRow(buff);
if (r == null) {
break;
}
list.add(r);
}
}
index++;
r = list.get(listIndex++);
}
return r;
}
private Row readRow(Data buff) {
if (buff.readByte() == 0) {
return null;
}
int mem = buff.readInt();
int columnCount = buff.readInt();
long key = buff.readLong();
int version = buff.readInt();
if (readUncached) {
key = 0;
}
boolean deleted = buff.readInt() == 1;
int sessionId = buff.readInt();
Value[] values = new Value[columnCount];
for (int i = 0; i < columnCount; i++) {
Value v;
if (buff.readByte() == 0) {
v = null;
} else {
v = buff.readValue();
if (v.isLinked()) {
// the table id is 0 if it was linked when writing
// a temporary entry
if (v.getTableId() == 0) {
session.unlinkAtCommit(v);
}
}
}
values[i] = v;
}
Row row = new Row(values, mem);
row.setKey(key);
row.setVersion(version);
row.setDeleted(deleted);
row.setSessionId(sessionId);
return row;
}
private void readRow(ResultDiskTape tape) {
int min = Constants.FILE_BLOCK_SIZE;
Data buff = rowBuff;
buff.reset();
file.readFully(buff.getBytes(), 0, min);
int len = buff.readInt();
buff.checkCapacity(len);
if (len - min > 0) {
file.readFully(buff.getBytes(), min, len - min);
}
tape.pos += len;
Value[] row = new Value[columnCount];
for (int k = 0; k < columnCount; k++) {
row[k] = buff.readValue();
}
tape.buffer.add(row);
}
private void removeRow(int i) {
index.getPageStore().logUndo(this, data);
written = false;
changeCount = index.getPageStore().getChangeCount();
if (!optimizeUpdate) {
readAllRows();
}
Row r = getRowAt(i);
if (r != null) {
memoryChange(false, r);
}
entryCount--;
if (entryCount < 0) {
DbException.throwInternalError();
}
if (firstOverflowPageId != 0) {
start -= 4;
freeOverflow();
firstOverflowPageId = 0;
overflowRowSize = 0;
rowRef = null;
}
int keyOffsetPairLen = 2 + Data.getVarLongLen(keys[i]);
int startNext = i > 0 ? offsets[i - 1] : index.getPageStore().getPageSize();
int rowLength = startNext - offsets[i];
if (optimizeUpdate) {
if (writtenData) {
byte[] d = data.getBytes();
int dataStart = offsets[entryCount];
System.arraycopy(d, dataStart, d, dataStart + rowLength, offsets[i] - dataStart);
Arrays.fill(d, dataStart, dataStart + rowLength, (byte) 0);
}
} else {
int clearStart = offsets[entryCount];
Arrays.fill(data.getBytes(), clearStart, clearStart + rowLength, (byte) 0);
}
start -= keyOffsetPairLen;
offsets = remove(offsets, entryCount + 1, i);
add(offsets, i, entryCount, rowLength);
keys = remove(keys, entryCount + 1, i);
rows = remove(rows, entryCount + 1, i);
}
ResultDiskBuffer(Session session, SortOrder sort, int columnCount) {
this.parent = null;
this.sort = sort;
this.columnCount = columnCount;
Database db = session.getDatabase();
rowBuff = Data.create(db, Constants.DEFAULT_PAGE_SIZE);
String fileName = db.createTempFile();
file = db.openFile(fileName, "rw", false);
file.setCheckedWriting(false);
file.seek(FileStore.HEADER_LENGTH);
if (sort != null) {
tapes = New.arrayList();
mainTape = null;
} else {
tapes = null;
mainTape = new ResultDiskTape();
mainTape.pos = FileStore.HEADER_LENGTH;
}
this.maxBufferSize = db.getSettings().largeResultBufferSize;
}
/**
* Write the data to the buffer.
*
* @param buff the source data
* @param off the offset in the source buffer
* @param len the number of bytes to write
* @return the number of bytes written
*/
int write(byte[] buff, int offset, int len) {
int max = Math.min(remaining, len);
data.write(buff, offset, max);
remaining -= max;
return max;
}
/**
* Read the next bytes from the buffer.
*
* @param startPos the position in the data page
* @param buff the target buffer
* @param off the offset in the target buffer
* @param len the number of bytes to read
*/
void read(int startPos, byte[] buff, int off, int len) {
System.arraycopy(data.getBytes(), startPos, buff, off, len);
}
/**
* A record is added to a table, or removed from a table.
*
* @param session the session
* @param tableId the table id
* @param row the row to add
* @param add true if the row is added, false if it is removed
*/
void logAddOrRemoveRow(Session session, int tableId, Row row, boolean add) {
if (trace.isDebugEnabled()) {
trace.debug(
"log "
+ (add ? "+" : "-")
+ " s: "
+ session.getId()
+ " table: "
+ tableId
+ " row: "
+ row);
}
session.addLogPos(logSectionId, logPos);
logPos++;
Data data = dataBuffer;
data.reset();
int columns = row.getColumnCount();
data.writeVarInt(columns);
data.checkCapacity(row.getByteCount(data));
if (session.isRedoLogBinaryEnabled()) {
for (int i = 0; i < columns; i++) {
data.writeValue(row.getValue(i));
}
} else {
for (int i = 0; i < columns; i++) {
Value v = row.getValue(i);
if (v.getType() == Value.BYTES) {
data.writeValue(ValueNull.INSTANCE);
} else {
data.writeValue(v);
}
}
}
Data buffer = getBuffer();
buffer.writeByte((byte) (add ? ADD : REMOVE));
buffer.writeVarInt(session.getId());
buffer.writeVarInt(tableId);
buffer.writeVarLong(row.getKey());
if (add) {
buffer.writeVarInt(data.length());
buffer.checkCapacity(data.length());
buffer.write(data.getBytes(), 0, data.length());
}
write(buffer);
}
@Override
public int addRows(ArrayList<Value[]> rows) {
if (sort != null) {
sort.sort(rows);
}
Data buff = rowBuff;
long start = file.getFilePointer();
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
int bufferLen = 0;
for (Value[] row : rows) {
buff.reset();
buff.writeInt(0);
for (int j = 0; j < columnCount; j++) {
Value v = row[j];
buff.checkCapacity(buff.getValueLen(v));
buff.writeValue(v);
}
buff.fillAligned();
int len = buff.length();
buff.setInt(0, len);
if (maxBufferSize > 0) {
buffer.write(buff.getBytes(), 0, len);
bufferLen += len;
if (bufferLen > maxBufferSize) {
byte[] data = buffer.toByteArray();
buffer.reset();
file.write(data, 0, data.length);
bufferLen = 0;
}
} else {
file.write(buff.getBytes(), 0, len);
}
}
if (bufferLen > 0) {
byte[] data = buffer.toByteArray();
file.write(data, 0, data.length);
}
if (sort != null) {
ResultDiskTape tape = new ResultDiskTape();
tape.start = start;
tape.end = file.getFilePointer();
tapes.add(tape);
} else {
mainTape.end = file.getFilePointer();
}
rowCount += rows.size();
return rowCount;
}
private Data getBuffer() {
if (writeBuffer.length() == 0) {
return writeBuffer;
}
return store.createData();
}