@Test
public void simpleWriteRead() {
byte primaryRingSizeInBits = 9;
byte byteRingSizeInBits = 18;
Pipe ring = new Pipe(new PipeConfig(primaryRingSizeInBits, byteRingSizeInBits, null, FROM));
ring.initBuffers();
int messageSize = FROM.fragDataSize[FRAG_LOC];
int varDataMax = (ring.byteMask / (ring.mask >> 1)) / messageSize;
int testSize = ((1 << primaryRingSizeInBits) / messageSize) - 1; // room for EOF
writeTestValue(ring, varDataMax, testSize);
// now read the data back
int FIELD_LOC =
FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
int k = testSize;
while (PipeReader.tryReadFragment(ring)) {
--k;
assertTrue(PipeReader.isNewMessage(ring));
int messageIdx = PipeReader.getMsgIdx(ring);
if (messageIdx < 0) {
return;
}
testReadValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
}
}
@Test
public void simpleWriteReadThreaded() {
final byte primaryRingSizeInBits = 7; // this ring is 2^7 eg 128
final byte byteRingSizeInBits = 16;
final Pipe ring =
new Pipe(new PipeConfig(primaryRingSizeInBits, byteRingSizeInBits, null, FROM));
ring.initBuffers();
final int messageSize = FROM.fragDataSize[FRAG_LOC];
final int varDataMax = (ring.byteMask / (ring.mask >> 1)) / messageSize;
final int testSize = (1 << primaryRingSizeInBits) / messageSize;
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
writeTestValue(ring, varDataMax, testSize);
}
});
t.start();
// now read the data back
int FIELD_LOC =
FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
int k = testSize;
while (k > 0) {
// This is the example code that one would normally use.
// System.err.println("content "+ring.contentRemaining(ring));
if (PipeReader.tryReadFragment(
ring)) { // this method releases old messages as needed and moves pointer up to the next
// fragment
k--; // count down all the expected messages so we stop this test at the right time
assertTrue(PipeReader.isNewMessage(ring));
int messageIdx = PipeReader.getMsgIdx(ring);
if (messageIdx < 0) {
return;
}
testReadValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
} else {
// unable to read so at this point
// we can do other work and try again soon
Thread.yield();
}
}
}
private void writeTestValue(Pipe ring, int blockSize, int testSize) {
int FIELD_LOC =
FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
assertTrue(0 == Pipe.contentRemaining(ring));
int j = testSize;
while (true) {
if (j == 0) {
PipeWriter.publishEOF(ring);
return; // done
}
if (PipeWriter.tryWriteFragment(
ring, FRAG_LOC)) { // returns true if there is room to write this fragment
int value = (--j * blockSize) / testSize;
PipeWriter.writeFloatAsIntBits(ring, FIELD_LOC, 1f / (float) value);
PipeWriter.publishWrites(ring); // must always publish the writes if message or fragment
} else {
// Unable to write because there is no room so do something else while we are waiting.
Thread.yield();
}
}
}
public OutputPipeInvocationHandler(Pipe outputRing, int msgIdx, Class<?> clazz) {
super(clazz.getMethods());
FieldReferenceOffsetManager from = Pipe.from(outputRing);
final Method[] methods = clazz.getMethods();
writers = new OutputPipeWriterMethod[MAX_METHODS];
int j = methods.length;
while (--j >= 0) {
final Method method = methods[j];
ProngTemplateField fieldAnnonation = method.getAnnotation(ProngTemplateField.class);
if (null != fieldAnnonation) {
int fieldLoc =
FieldReferenceOffsetManager.lookupFieldLocator(fieldAnnonation.fieldId(), msgIdx, from);
int key = buildKey(this, method.getName());
if (null != writers[key]) {
throw new UnsupportedOperationException();
}
writers[key] =
OutputPipeWriterMethod.buildWriteForYourType(
outputRing,
fieldAnnonation.decimalPlaces(),
fieldLoc,
(fieldLoc >> FieldReferenceOffsetManager.RW_FIELD_OFF_BITS)
& TokenBuilder.MASK_TYPE,
from);
}
}
}
@Before
public void createTestFile() {
rb = new Pipe(new PipeConfig((byte) 21, (byte) 7, null, FieldReferenceOffsetManager.RAW_BYTES));
rb.initBuffers();
try {
// build a bunch of data
int i = 1 << 26;
byte[] temp = new byte[i];
while (--i >= 0) {
if ((i & 0x3F) == 0) {
temp[i] = '\n';
} else {
if ((((i + 1) & 0x7) == 0) && (((i + 9) & 0x1F) == 0)) {
temp[i] = '\\';
} else {
if ((i & 0x7) == 0) {
temp[i] = ',';
} else {
temp[i] = (byte) ('0' + (int) (0x1F & i));
}
}
}
}
// System.err.println(new String(temp));
// write the data to the file
File f = File.createTempFile(this.getClass().getSimpleName(), "test");
f.deleteOnExit(); // but do keep it arround while we do our test
FileOutputStream out = new FileOutputStream(f);
out.write(temp);
out.close();
testFile = f;
} catch (IOException e) {
e.printStackTrace();
fail();
}
}
/**
* Experimental parser built to leverage multiple cores and keep up with the speed of modern SSDs
*
* @param fileChannel
* @throws IOException
*/
public void extract(FileChannel fileChannel) throws IOException {
MappedByteBuffer mappedBuffer;
long fileSize = fileChannel.size();
long position = 0;
int tailPadding = 8; // needed to cover the transition
long blockSize = 1 << 25;
TypeExtractor typeExtractor = new TypeExtractor(true /* force ASCII */);
RecordFieldExtractor rfe = new RecordFieldExtractor();
mappedBuffer =
fileChannel.map(
FileChannel.MapMode.READ_ONLY, position, Math.min(blockSize, fileSize - position));
int padding = tailPadding;
do {
if (mappedBuffer.limit() + position == fileSize) {
padding = 0;
}
int pos = 0;
Pipe.setValue(
rb.structuredLayoutRingBuffer,
rb.mask,
Pipe.getWorkingHeadPositionObject(rb).value++,
pos);
int tokenCount = 0;
int c = 0;
int j = mappedBuffer.remaining() - padding;
do {
// walk over the data while we have this section mapped.
c++;
byte b = (byte) mappedBuffer.get();
// RecordFieldExtractor.appendContent(rfe, b); //TOO much work here must do on reading
// thread.
// TODO: check the field type sums
// TODO: zero copy but we need to discover tokens
// splits on returns, commas, dots and many other punctuation
if (b < 48) {
// System.err.println("char :"+b);
// what mask can be built to combine the byte we are after.
// allTheBits++; //do something
pos = mappedBuffer.position();
Pipe.setValue(
rb.structuredLayoutRingBuffer,
rb.mask,
Pipe.getWorkingHeadPositionObject(rb).value++,
pos);
if ((++tokenCount & 0xF) == 0) {
Pipe.publishWrites(rb);
}
// rb.reset();
}
} while (--j > 0);
// this tokenizer assumes that the file ends with a field delimiter so the last record gets
// flushed.
// TODO: need to wait for threads to finish before swapping to new page or have multiple pages
// to swap in/out
// only increment by exactly how many bytes were read assuming we started at zero
// can only cut at the last known record start
position += c;
System.out.println("bytes read so far:" + position);
mappedBuffer =
fileChannel.map(
FileChannel.MapMode.READ_ONLY, position, Math.min(blockSize, fileSize - position));
} while (position < fileSize);
}