/**
* Create a BLOB value from a stream.
*
* @param in the input stream
* @param length the number of characters to read, or -1 for no limit
* @param handler the data handler
* @return the lob value
*/
private static ValueLob createBlob(InputStream in, long length, DataHandler handler) {
try {
if (handler == null) {
byte[] data = IOUtils.readBytesAndClose(in, (int) length);
return createSmallLob(Value.BLOB, data);
}
long remaining = Long.MAX_VALUE;
boolean compress = handler.getLobCompressionAlgorithm(Value.BLOB) != null;
if (length >= 0 && length < remaining) {
remaining = length;
}
int len = getBufferSize(handler, compress, remaining);
byte[] buff;
if (len >= Integer.MAX_VALUE) {
buff = IOUtils.readBytesAndClose(in, -1);
len = buff.length;
} else {
buff = DataUtils.newBytes(len);
len = IOUtils.readFully(in, buff, len);
}
if (len <= handler.getMaxLengthInplaceLob()) {
byte[] small = DataUtils.newBytes(len);
System.arraycopy(buff, 0, small, 0, len);
return ValueLob.createSmallLob(Value.BLOB, small);
}
ValueLob lob = new ValueLob(Value.BLOB, null);
lob.createFromStream(buff, len, in, remaining, handler);
return lob;
} catch (IOException e) {
throw DbException.convertIOException(e, null);
}
}
/**
* Create a CLOB value from a stream.
*
* @param in the reader
* @param length the number of characters to read, or -1 for no limit
* @param handler the data handler
* @return the lob value
*/
private static ValueLob createClob(Reader in, long length, DataHandler handler) {
try {
if (handler == null) {
String s = IOUtils.readStringAndClose(in, (int) length);
return createSmallLob(Value.CLOB, s.getBytes(Constants.UTF8));
}
boolean compress = handler.getLobCompressionAlgorithm(Value.CLOB) != null;
long remaining = Long.MAX_VALUE;
if (length >= 0 && length < remaining) {
remaining = length;
}
int len = getBufferSize(handler, compress, remaining);
char[] buff;
if (len >= Integer.MAX_VALUE) {
String data = IOUtils.readStringAndClose(in, -1);
buff = data.toCharArray();
len = buff.length;
} else {
buff = new char[len];
len = IOUtils.readFully(in, buff, len);
}
if (len <= handler.getMaxLengthInplaceLob()) {
byte[] small = new String(buff, 0, len).getBytes(Constants.UTF8);
return ValueLob.createSmallLob(Value.CLOB, small);
}
ValueLob lob = new ValueLob(Value.CLOB, null);
lob.createFromReader(buff, len, in, remaining, handler);
return lob;
} catch (IOException e) {
throw DbException.convertIOException(e, null);
}
}
private static synchronized void deleteFile(DataHandler handler, String fileName) {
// synchronize on the database, to avoid concurrent temp file creation /
// deletion / backup
synchronized (handler.getLobSyncObject()) {
FileUtils.delete(fileName);
}
}
private static void invalidateFileList(DataHandler h, String dir) {
SmallLRUCache<String, String[]> cache = h.getLobFileListCache();
if (cache != null) {
synchronized (cache) {
cache.remove(dir);
}
}
}
private static void copyFileTo(DataHandler h, String sourceFileName, String targetFileName) {
synchronized (h.getLobSyncObject()) {
try {
IOUtils.copyFiles(sourceFileName, targetFileName);
} catch (IOException e) {
throw DbException.convertIOException(e, null);
}
}
}
private static String getFileName(DataHandler handler, int tableId, int objectId) {
if (SysProperties.CHECK && tableId == 0 && objectId == 0) {
DbException.throwInternalError("0 LOB");
}
String table = tableId < 0 ? ".temp" : ".t" + tableId;
return getFileNamePrefix(handler.getDatabasePath(), objectId)
+ table
+ Constants.SUFFIX_LOB_FILE;
}
/**
* Store the lob data to a file if the size of the buffer is larger than the maximum size for an
* in-place lob.
*
* @param h the data handler
*/
public void convertToFileIfRequired(DataHandler h) {
try {
if (small != null && small.length > h.getMaxLengthInplaceLob()) {
boolean compress = h.getLobCompressionAlgorithm(type) != null;
int len = getBufferSize(h, compress, Long.MAX_VALUE);
int tabId = tableId;
if (type == Value.BLOB) {
createFromStream(DataUtils.newBytes(len), 0, getInputStream(), Long.MAX_VALUE, h);
} else {
createFromReader(new char[len], 0, getReader(), Long.MAX_VALUE, h);
}
Value v2 = link(h, tabId);
if (SysProperties.CHECK && v2 != this) {
DbException.throwInternalError();
}
}
} catch (IOException e) {
throw DbException.convertIOException(e, null);
}
}
@Override
public InputStream getInputStream() {
if (fileName == null) {
return new ByteArrayInputStream(small);
}
FileStore store = handler.openFile(fileName, "r", true);
boolean alwaysClose = SysProperties.LOB_CLOSE_BETWEEN_READS;
return new BufferedInputStream(
new FileStoreInputStream(store, handler, compressed, alwaysClose),
Constants.IO_BUFFER_SIZE);
}
private FileStoreOutputStream initLarge(DataHandler h) {
this.handler = h;
this.tableId = 0;
this.linked = false;
this.precision = 0;
this.small = null;
this.hash = 0;
String compressionAlgorithm = h.getLobCompressionAlgorithm(type);
this.compressed = compressionAlgorithm != null;
synchronized (h) {
String path = h.getDatabasePath();
if ((path != null) && (path.length() == 0)) {
path =
new File(Utils.getProperty("java.io.tmpdir", "."), SysProperties.PREFIX_TEMP_FILE)
.getAbsolutePath();
}
objectId = getNewObjectId(h);
fileName = getFileNamePrefix(path, objectId) + Constants.SUFFIX_TEMP_FILE;
tempFile = h.openFile(fileName, "rw", false);
tempFile.autoDelete();
}
FileStoreOutputStream out = new FileStoreOutputStream(tempFile, h, compressionAlgorithm);
return out;
}
private static String[] getFileList(DataHandler h, String dir) {
SmallLRUCache<String, String[]> cache = h.getLobFileListCache();
String[] list;
if (cache == null) {
list = FileUtils.newDirectoryStream(dir).toArray(new String[0]);
} else {
synchronized (cache) {
list = cache.get(dir);
if (list == null) {
list = FileUtils.newDirectoryStream(dir).toArray(new String[0]);
cache.put(dir, list);
}
}
}
return list;
}
private static int getBufferSize(DataHandler handler, boolean compress, long remaining) {
if (remaining < 0 || remaining > Integer.MAX_VALUE) {
remaining = Integer.MAX_VALUE;
}
int inplace = handler.getMaxLengthInplaceLob();
long m = compress ? Constants.IO_BUFFER_SIZE_COMPRESS : Constants.IO_BUFFER_SIZE;
if (m < remaining && m <= inplace) {
// using "1L" to force long arithmetic
m = Math.min(remaining, inplace + 1L);
// the buffer size must be bigger than the inplace lob, otherwise we
// can't know if it must be stored in-place or not
m = MathUtils.roundUpLong(m, Constants.IO_BUFFER_SIZE);
}
m = Math.min(remaining, m);
m = MathUtils.convertLongToInt(m);
if (m < 0) {
m = Integer.MAX_VALUE;
}
return (int) m;
}
private void createFromStream(byte[] buff, int len, InputStream in, long remaining, DataHandler h)
throws IOException {
FileStoreOutputStream out = initLarge(h);
boolean compress = h.getLobCompressionAlgorithm(Value.BLOB) != null;
try {
while (true) {
precision += len;
out.write(buff, 0, len);
remaining -= len;
if (remaining <= 0) {
break;
}
len = getBufferSize(h, compress, remaining);
len = IOUtils.readFully(in, buff, len);
if (len <= 0) {
break;
}
}
} finally {
out.close();
}
}
private void createFromReader(char[] buff, int len, Reader in, long remaining, DataHandler h)
throws IOException {
FileStoreOutputStream out = initLarge(h);
boolean compress = h.getLobCompressionAlgorithm(Value.CLOB) != null;
try {
while (true) {
precision += len;
byte[] b = new String(buff, 0, len).getBytes(Constants.UTF8);
out.write(b, 0, b.length);
remaining -= len;
if (remaining <= 0) {
break;
}
len = getBufferSize(h, compress, remaining);
len = IOUtils.readFully(in, buff, len);
if (len == 0) {
break;
}
}
} finally {
out.close();
}
}
private static synchronized void renameFile(DataHandler handler, String oldName, String newName) {
synchronized (handler.getLobSyncObject()) {
FileUtils.move(oldName, newName);
}
}
private static int getNewObjectId(DataHandler h) {
String path = h.getDatabasePath();
if ((path != null) && (path.length() == 0)) {
path =
new File(Utils.getProperty("java.io.tmpdir", "."), SysProperties.PREFIX_TEMP_FILE)
.getAbsolutePath();
}
int newId = 0;
int lobsPerDir = SysProperties.LOB_FILES_PER_DIRECTORY;
while (true) {
String dir = getFileNamePrefix(path, newId);
String[] list = getFileList(h, dir);
int fileCount = 0;
boolean[] used = new boolean[lobsPerDir];
for (String name : list) {
if (name.endsWith(Constants.SUFFIX_DB_FILE)) {
name = FileUtils.getName(name);
String n = name.substring(0, name.indexOf('.'));
int id;
try {
id = Integer.parseInt(n);
} catch (NumberFormatException e) {
id = -1;
}
if (id > 0) {
fileCount++;
used[id % lobsPerDir] = true;
}
}
}
int fileId = -1;
if (fileCount < lobsPerDir) {
for (int i = 1; i < lobsPerDir; i++) {
if (!used[i]) {
fileId = i;
break;
}
}
}
if (fileId > 0) {
newId += fileId;
invalidateFileList(h, dir);
break;
}
if (newId > Integer.MAX_VALUE / lobsPerDir) {
// this directory path is full: start from zero
newId = 0;
dirCounter = MathUtils.randomInt(lobsPerDir - 1) * lobsPerDir;
} else {
// calculate the directory.
// start with 1 (otherwise we don't know the number of
// directories).
// it doesn't really matter what directory is used, it might as
// well be random (but that would generate more directories):
// int dirId = RandomUtils.nextInt(lobsPerDir - 1) + 1;
int dirId = (dirCounter++ / (lobsPerDir - 1)) + 1;
newId = newId * lobsPerDir;
newId += dirId * lobsPerDir;
}
}
return newId;
}
