コード例 #1
0
  @Test
  public void simpleWriteRead() {

    byte primaryRingSizeInBits = 9;
    byte byteRingSizeInBits = 18;

    Pipe ring = new Pipe(new PipeConfig(primaryRingSizeInBits, byteRingSizeInBits, null, FROM));
    ring.initBuffers();

    int messageSize = FROM.fragDataSize[FRAG_LOC];

    int varDataMax = (ring.byteMask / (ring.mask >> 1)) / messageSize;
    int testSize = ((1 << primaryRingSizeInBits) / messageSize) - 1; // room for EOF

    writeTestValue(ring, varDataMax, testSize);

    // now read the data back
    int FIELD_LOC =
        FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);

    int k = testSize;
    while (PipeReader.tryReadFragment(ring)) {

      --k;
      assertTrue(PipeReader.isNewMessage(ring));
      int messageIdx = PipeReader.getMsgIdx(ring);
      if (messageIdx < 0) {
        return;
      }
      testReadValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);
    }
  }
コード例 #2
0
  @Test
  public void simpleWriteReadThreaded() {

    final byte primaryRingSizeInBits = 7; // this ring is 2^7 eg 128
    final byte byteRingSizeInBits = 16;
    final Pipe ring =
        new Pipe(new PipeConfig(primaryRingSizeInBits, byteRingSizeInBits, null, FROM));
    ring.initBuffers();

    final int messageSize = FROM.fragDataSize[FRAG_LOC];

    final int varDataMax = (ring.byteMask / (ring.mask >> 1)) / messageSize;
    final int testSize = (1 << primaryRingSizeInBits) / messageSize;

    Thread t =
        new Thread(
            new Runnable() {

              @Override
              public void run() {
                writeTestValue(ring, varDataMax, testSize);
              }
            });
    t.start();

    // now read the data back

    int FIELD_LOC =
        FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);

    int k = testSize;
    while (k > 0) {

      // This is the example code that one would normally use.

      // System.err.println("content "+ring.contentRemaining(ring));
      if (PipeReader.tryReadFragment(
          ring)) { // this method releases old messages as needed and moves pointer up to the next
        // fragment
        k--; // count down all the expected messages so we stop this test at the right time

        assertTrue(PipeReader.isNewMessage(ring));
        int messageIdx = PipeReader.getMsgIdx(ring);
        if (messageIdx < 0) {
          return;
        }
        testReadValue(ring, varDataMax, testSize, FIELD_LOC, k, messageIdx);

      } else {
        // unable to read so at this point
        // we can do other work and try again soon
        Thread.yield();
      }
    }
  }
コード例 #3
0
  private void writeTestValue(Pipe ring, int blockSize, int testSize) {

    int FIELD_LOC =
        FieldReferenceOffsetManager.lookupFieldLocator(SINGLE_MESSAGE_NAMES[0], FRAG_LOC, FROM);
    assertTrue(0 == Pipe.contentRemaining(ring));
    int j = testSize;
    while (true) {

      if (j == 0) {
        PipeWriter.publishEOF(ring);
        return; // done
      }

      if (PipeWriter.tryWriteFragment(
          ring, FRAG_LOC)) { // returns true if there is room to write this fragment

        int value = (--j * blockSize) / testSize;
        PipeWriter.writeFloatAsIntBits(ring, FIELD_LOC, 1f / (float) value);
        PipeWriter.publishWrites(ring); // must always publish the writes if message or fragment

      } else {
        // Unable to write because there is no room so do something else while we are waiting.
        Thread.yield();
      }
    }
  }
コード例 #4
0
  public OutputPipeInvocationHandler(Pipe outputRing, int msgIdx, Class<?> clazz) {
    super(clazz.getMethods());

    FieldReferenceOffsetManager from = Pipe.from(outputRing);
    final Method[] methods = clazz.getMethods();

    writers = new OutputPipeWriterMethod[MAX_METHODS];
    int j = methods.length;
    while (--j >= 0) {
      final Method method = methods[j];
      ProngTemplateField fieldAnnonation = method.getAnnotation(ProngTemplateField.class);
      if (null != fieldAnnonation) {

        int fieldLoc =
            FieldReferenceOffsetManager.lookupFieldLocator(fieldAnnonation.fieldId(), msgIdx, from);

        int key = buildKey(this, method.getName());
        if (null != writers[key]) {
          throw new UnsupportedOperationException();
        }
        writers[key] =
            OutputPipeWriterMethod.buildWriteForYourType(
                outputRing,
                fieldAnnonation.decimalPlaces(),
                fieldLoc,
                (fieldLoc >> FieldReferenceOffsetManager.RW_FIELD_OFF_BITS)
                    & TokenBuilder.MASK_TYPE,
                from);
      }
    }
  }
コード例 #5
0
  @Before
  public void createTestFile() {

    rb = new Pipe(new PipeConfig((byte) 21, (byte) 7, null, FieldReferenceOffsetManager.RAW_BYTES));
    rb.initBuffers();
    try {

      // build a bunch of data
      int i = 1 << 26;
      byte[] temp = new byte[i];
      while (--i >= 0) {
        if ((i & 0x3F) == 0) {
          temp[i] = '\n';
        } else {
          if ((((i + 1) & 0x7) == 0) && (((i + 9) & 0x1F) == 0)) {
            temp[i] = '\\';
          } else {
            if ((i & 0x7) == 0) {
              temp[i] = ',';
            } else {
              temp[i] = (byte) ('0' + (int) (0x1F & i));
            }
          }
        }
      }

      // System.err.println(new String(temp));

      // write the data to the file
      File f = File.createTempFile(this.getClass().getSimpleName(), "test");
      f.deleteOnExit(); // but do keep it arround while we do our test

      FileOutputStream out = new FileOutputStream(f);
      out.write(temp);
      out.close();

      testFile = f;
    } catch (IOException e) {
      e.printStackTrace();
      fail();
    }
  }
コード例 #6
0
  /**
   * Experimental parser built to leverage multiple cores and keep up with the speed of modern SSDs
   *
   * @param fileChannel
   * @throws IOException
   */
  public void extract(FileChannel fileChannel) throws IOException {
    MappedByteBuffer mappedBuffer;

    long fileSize = fileChannel.size();
    long position = 0;
    int tailPadding = 8; // needed to cover the transition
    long blockSize = 1 << 25;

    TypeExtractor typeExtractor = new TypeExtractor(true /* force ASCII */);
    RecordFieldExtractor rfe = new RecordFieldExtractor();

    mappedBuffer =
        fileChannel.map(
            FileChannel.MapMode.READ_ONLY, position, Math.min(blockSize, fileSize - position));
    int padding = tailPadding;
    do {

      if (mappedBuffer.limit() + position == fileSize) {
        padding = 0;
      }

      int pos = 0;

      Pipe.setValue(
          rb.structuredLayoutRingBuffer,
          rb.mask,
          Pipe.getWorkingHeadPositionObject(rb).value++,
          pos);

      int tokenCount = 0;
      int c = 0;

      int j = mappedBuffer.remaining() - padding;
      do {
        // walk over the data while we have this section mapped.
        c++;

        byte b = (byte) mappedBuffer.get();

        //      	RecordFieldExtractor.appendContent(rfe, b); //TOO much work here must do on reading
        // thread.

        // TODO: check the field type sums
        // TODO: zero copy but we need to discover tokens

        // splits on returns, commas, dots and many other punctuation
        if (b < 48) {
          // System.err.println("char :"+b);

          // what mask can be built to combine the byte we are after.

          //	allTheBits++; //do something

          pos = mappedBuffer.position();
          Pipe.setValue(
              rb.structuredLayoutRingBuffer,
              rb.mask,
              Pipe.getWorkingHeadPositionObject(rb).value++,
              pos);

          if ((++tokenCount & 0xF) == 0) {
            Pipe.publishWrites(rb);
          }

          //	rb.reset();

        }

      } while (--j > 0);

      // this tokenizer assumes that the file ends with a field delimiter so the last record gets
      // flushed.

      // TODO: need to wait for threads to finish before swapping to new page or have multiple pages
      // to swap in/out

      // only increment by exactly how many bytes were read assuming we started at zero
      // can only cut at the last known record start
      position += c;

      System.out.println("bytes read so far:" + position);

      mappedBuffer =
          fileChannel.map(
              FileChannel.MapMode.READ_ONLY, position, Math.min(blockSize, fileSize - position));

    } while (position < fileSize);
  }