public void testRun() throws Exception {
String wsName = "test";
DefaultCacheManager dcm = new DefaultCacheManager();
Cache<String, Metadata> metaCache = dcm.getCache("meta");
Cache<String, byte[]> dataCache = dcm.getCache("data");
GridFilesystem gfs = new GridFilesystem(dataCache, metaCache);
BuildContext bcontext =
DirectoryBuilder.newDirectoryInstance(metaCache, dataCache, metaCache, wsName);
bcontext.chunkSize(1024 * 1024);
Directory directory = bcontext.create();
Central central = CentralConfig.oldFromDir(directory).build();
central.newIndexer().index(IndexJobs.create("/bleujin", 10));
central.newSearcher().createRequest("").find().debugPrint();
OutputStream output = gfs.getOutput("/test.data");
IOUtil.copyNClose(new StringInputStream("hello bleujin"), output);
Debug.line(IOUtil.toStringWithClose(gfs.getInput("/test.data")));
central.newSearcher().createRequest("").find().debugPrint();
File root = gfs.getFile("/");
viewFile(root);
dcm.stop();
}
@Override
Directory createDirectory(Cache cache) {
return DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(CHUNK_SIZE)
.overrideSegmentReadLocker(new LocalLockMergingSegmentReadLocker(cache, INDEX_NAME))
.create();
}
public void multipleFlushTest() throws IOException {
final String filename = "longFile.writtenInMultipleFlushes";
final int bufferSize = 300;
Cache cache = cacheManager.getCache();
cache.clear();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(13)
.create();
byte[] manyBytes = fillBytes(bufferSize);
IndexOutput indexOutput = dir.createOutput(filename);
for (int i = 0; i < 10; i++) {
indexOutput.writeBytes(manyBytes, bufferSize);
indexOutput.flush();
}
indexOutput.close();
IndexInput input = dir.openInput(filename);
final int finalSize = (10 * bufferSize);
AssertJUnit.assertEquals(finalSize, input.length());
final byte[] resultingBuffer = new byte[finalSize];
input.readBytes(resultingBuffer, 0, finalSize);
int index = 0;
for (int i = 0; i < 10; i++) {
for (int j = 0; j < bufferSize; j++)
AssertJUnit.assertEquals(resultingBuffer[index++], manyBytes[j]);
}
}
@Test
public void testChunkBordersOnInfinispan() throws IOException {
Cache cache = cacheManager.getCache();
cache.clear();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(13)
.create();
testChunkBorders(dir, cache);
cache.clear();
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testFailureOfOverrideWriteLocker() throws IOException {
Directory dir = null;
try {
dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(BUFFER_SIZE)
.overrideWriteLocker(null)
.create();
} finally {
if (dir != null) dir.close();
}
}
@Test(
expectedExceptions = IllegalArgumentException.class,
expectedExceptionsMessageRegExp = "chunkSize must be a positive integer")
public void testInitWithInvalidChunkSize() throws IOException {
Directory dir = null;
try {
dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(0)
.create();
} finally {
if (dir != null) dir.close();
}
}
public void testWriteChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
IndexOutput io = dir.createOutput("MyNewFile.txt");
io.writeByte((byte) 66);
io.writeByte((byte) 69);
io.flush();
io.close();
assert dir.fileExists("MyNewFile.txt");
assert null != cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
// test contents by reading:
byte[] buf = new byte[9];
IndexInput ii = dir.openInput("MyNewFile.txt");
ii.readBytes(buf, 0, (int) ii.length());
ii.close();
assert new String(new byte[] {66, 69}).equals(new String(buf).trim());
String testText =
"This is some rubbish again that will span more than one chunk - one hopes. Who knows, maybe even three or four chunks.";
io = dir.createOutput("MyNewFile.txt");
io.seek(0);
io.writeBytes(testText.getBytes(), 0, testText.length());
io.close();
// now compare.
byte[] chunk1 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
byte[] chunk2 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 1, BUFFER_SIZE));
assert null != chunk1;
assert null != chunk2;
assert testText.equals(new String(chunk1) + new String(chunk2).trim());
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
@Test
public void testRunningOnMultipleCaches() throws IOException {
assert metadataCache != chunkCache;
assert chunkCache != lockCache;
assert lockCache != metadataCache;
Directory dir =
DirectoryBuilder.newDirectoryInstance(metadataCache, chunkCache, lockCache, "testingIndex")
.chunkSize(100)
.create();
writeTextToIndex(dir, 0, "hello world");
assertTextIsFoundInIds(dir, "hello", 0);
writeTextToIndex(dir, 1, "hello solar system");
assertTextIsFoundInIds(dir, "hello", 0, 1);
assertTextIsFoundInIds(dir, "system", 1);
optimizeIndex(dir);
assertTextIsFoundInIds(dir, "hello", 0, 1);
dir.close();
}
@Test
public void testGetIndexNameAndToString() throws IOException {
Cache cache = cacheManager.getCache();
Directory dir = null;
try {
dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(BUFFER_SIZE)
.create();
AssertJUnit.assertEquals(INDEX_NAME, ((DirectoryLuceneV4) dir).getIndexName());
AssertJUnit.assertEquals(
"InfinispanDirectory{indexName=\'" + INDEX_NAME + "\'}", dir.toString());
} finally {
if (dir != null) dir.close();
}
}
@Test
public void testWriteChunksDefaultChunks() throws Exception {
Cache cache = cacheManager.getCache();
Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME).create();
final String testText = "This is some rubbish";
final byte[] testTextAsBytes = testText.getBytes();
IndexOutput io = dir.createOutput("MyNewFile.txt");
io.writeByte((byte) 1);
io.writeByte((byte) 2);
io.writeByte((byte) 3);
io.writeBytes(testTextAsBytes, testTextAsBytes.length);
io.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
FileCacheKey fileCacheKey = new FileCacheKey(INDEXNAME, "MyNewFile.txt");
assert null != cache.get(fileCacheKey);
FileMetadata metadata = (FileMetadata) cache.get(fileCacheKey);
AssertJUnit.assertEquals(testTextAsBytes.length + 3, metadata.getSize());
assert null
!= cache.get(
new ChunkCacheKey(
INDEXNAME, "MyNewFile.txt", 0, DirectoryBuilderImpl.DEFAULT_BUFFER_SIZE));
// test contents by reading:
IndexInput ii = dir.openInput("MyNewFile.txt");
assert ii.readByte() == 1;
assert ii.readByte() == 2;
assert ii.readByte() == 3;
byte[] buf = new byte[testTextAsBytes.length];
ii.readBytes(buf, 0, testTextAsBytes.length);
ii.close();
assert testText.equals(new String(buf).trim());
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public void testOverrideWriteLocker() throws IOException {
Directory dir = null;
try {
dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(BUFFER_SIZE)
.overrideWriteLocker(
new LockFactory() {
@Override
public Lock makeLock(String lockName) {
return null;
}
@Override
public void clearLock(String lockName) throws IOException {}
})
.create();
AssertJUnit.assertEquals(0, dir.listAll().length);
} finally {
if (dir != null) dir.close();
}
}
@Test
public void testReadWholeFile() throws IOException {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
verifyOnBuffer("SingleChunk.txt", 61, BUFFER_SIZE, cache, dir, 15);
final int VERY_BIG_FILE_SIZE = 10000;
assert BUFFER_SIZE < VERY_BIG_FILE_SIZE;
verifyOnBuffer("MultipleChunks.txt", VERY_BIG_FILE_SIZE, BUFFER_SIZE, cache, dir, 33);
final int LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE = 256;
assert (LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE % BUFFER_SIZE) == 0;
verifyOnBuffer(
"LastChunkFilled.txt", LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE, BUFFER_SIZE, cache, dir, 11);
assertHasNChunks(4, cache, INDEXNAME, "LastChunkFilled.txt.bak", BUFFER_SIZE);
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
final int LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE = 257;
assert (LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE % BUFFER_SIZE) == 1;
verifyOnBuffer(
"LonelyByteInLastChunk.txt",
LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE,
BUFFER_SIZE,
cache,
dir,
12);
assertHasNChunks(5, cache, INDEXNAME, "LonelyByteInLastChunk.txt.bak", BUFFER_SIZE);
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
@Test
public void testReadRandomSampleFile() throws IOException {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
final int FILE_SIZE = 1000;
assert BUFFER_SIZE < FILE_SIZE;
createFileWithRepeatableContent(dir, "RandomSampleFile.txt", FILE_SIZE);
IndexInput indexInput = dir.openInput("RandomSampleFile.txt");
assert indexInput.length() == FILE_SIZE;
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
Random r = new Random();
long seekPoint = 0;
// Now it reads some random byte and it compares to the expected byte
for (int i = 0; i < FILE_SIZE; i++) {
if (seekPoint == i) {
byte expectedByte = bytesGenerator.nextByte();
byte actualByte = indexInput.readByte();
assert expectedByte == actualByte;
seekPoint = indexInput.getFilePointer() + r.nextInt(10);
indexInput.seek(seekPoint);
} else {
bytesGenerator.nextByte();
}
}
indexInput.close();
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
@Override
public void start() {
try {
String name = ctx.getCache().getName();
EmbeddedCacheManager cacheManager = ctx.getCache().getCacheManager();
Cache<?, ?> metaCache = cacheManager.getCache(name + "-meta");
Cache<?, ?> dataCache = cacheManager.getCache(name + "-chunk");
BuildContext bcontext =
DirectoryBuilder.newDirectoryInstance(metaCache, dataCache, metaCache, name);
bcontext.chunkSize(1024 * 1024);
Directory directory = bcontext.create();
this.central =
CentralConfig.oldFromDir(directory)
.indexConfigBuilder()
.executorService(new WithinThreadExecutor())
.build();
this.configuration.store(this);
} catch (Exception e) {
throw new PersistenceException(e);
}
}
@Test
public void testWriteUsingSeekMethod() throws IOException {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
String fileName = "SomeText.txt";
IndexOutput io = dir.createOutput(fileName);
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
// It writes repeatable text
final int REPEATABLE_BUFFER_SIZE = 1501;
for (int i = 0; i < REPEATABLE_BUFFER_SIZE; i++) {
io.writeByte(bytesGenerator.nextByte());
}
io.flush();
assert io.length() == REPEATABLE_BUFFER_SIZE;
// Text to write on file with repeatable text
final String someText = "This is some text";
final byte[] someTextAsBytes = someText.getBytes();
// 4 points in random order where writing someText: at begin of file, at end of file, within a
// single chunk,
// between 2 chunks
final int[] pointers = {0, 635, REPEATABLE_BUFFER_SIZE, 135};
for (int i = 0; i < pointers.length; i++) {
io.seek(pointers[i]);
io.writeBytes(someTextAsBytes, someTextAsBytes.length);
}
io.close();
bytesGenerator.reset();
final long finalSize = REPEATABLE_BUFFER_SIZE + someTextAsBytes.length;
assert io.length() == finalSize;
assert io.length()
== DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, fileName), cache);
int indexPointer = 0;
Arrays.sort(pointers);
byte[] buffer = null;
int chunkIndex = -1;
// now testing the stream is equal to the produced repeatable but including the edits at pointed
// positions
for (int i = 0; i < REPEATABLE_BUFFER_SIZE + someTextAsBytes.length; i++) {
if (i % BUFFER_SIZE == 0) {
buffer =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, fileName, ++chunkIndex, BUFFER_SIZE));
}
byte predictableByte = bytesGenerator.nextByte();
if (i < pointers[indexPointer]) {
// Assert predictable text
AssertJUnit.assertEquals(predictableByte, buffer[i % BUFFER_SIZE]);
} else if (pointers[indexPointer] <= i
&& i < pointers[indexPointer] + someTextAsBytes.length) {
// Assert someText
AssertJUnit.assertEquals(
someTextAsBytes[i - pointers[indexPointer]], buffer[i % BUFFER_SIZE]);
}
if (i == pointers[indexPointer] + someTextAsBytes.length) {
// Change pointer
indexPointer++;
}
}
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
@Test(enabled = false)
public void testReadChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
// create file headers
FileMetadata file1 = new FileMetadata(5);
FileCacheKey key1 = new FileCacheKey(INDEXNAME, "Hello.txt");
cache.put(key1, file1);
FileMetadata file2 = new FileMetadata(5);
FileCacheKey key2 = new FileCacheKey(INDEXNAME, "World.txt");
cache.put(key2, file2);
// byte array for Hello.txt
String helloText = "Hello world. This is some text.";
cache.put(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE), helloText.getBytes());
// byte array for World.txt - should be in at least 2 chunks.
String worldText =
"This String should contain more than sixty four characters but less than one hundred and twenty eight.";
assert worldText.getBytes().length > BUFFER_SIZE;
assert worldText.getBytes().length < (2 * BUFFER_SIZE);
byte[] buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), 0, buf, 0, BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE), buf);
String part1 = new String(buf);
buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), BUFFER_SIZE, buf, 0, worldText.length() - BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE), buf);
String part2 = new String(buf);
// make sure the generated bytes do add up!
AssertJUnit.assertEquals(part1 + part2.trim(), worldText);
file1.setSize(helloText.length());
file2.setSize(worldText.length());
Set<String> s = new HashSet<String>();
s.add("Hello.txt");
s.add("World.txt");
Set other = new HashSet(Arrays.asList(dir.listAll()));
// ok, file listing works.
AssertJUnit.assertEquals(s, other);
IndexInput ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(worldText);
// now with buffered reading
ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
baos = new ByteArrayOutputStream();
long toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.deleteFile("Hello.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
Object ob1 = cache.get(new FileCacheKey(INDEXNAME, "World.txt"));
Object ob2 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE));
Object ob3 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE));
((DirectoryExtensions) dir).renameFile("World.txt", "HelloWorld.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 1, BUFFER_SIZE));
assert cache.get(new FileCacheKey(INDEXNAME, "HelloWorld.txt")).equals(ob1);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 0, BUFFER_SIZE)).equals(ob2);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 1, BUFFER_SIZE)).equals(ob3);
// test that contents survives a move
ii = dir.openInput("HelloWorld.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
