Exemplo n.º 1
0
  @Test
  public void testBufferCallbackUniqueBuffers() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    final int NUMBER_LINES = 1000;
    final int SIZE = 512;

    controller.open(fileName, 1000);

    controller.fill(0, 1, NUMBER_LINES * SIZE, (byte) 'j');

    final ArrayList<ByteBuffer> buffers = new ArrayList<ByteBuffer>();

    BufferCallback bufferCallback =
        new BufferCallback() {
          public void bufferDone(final ByteBuffer buffer) {
            buffers.add(buffer);
          }
        };

    controller.setBufferCallback(bufferCallback);

    CountDownLatch latch = new CountDownLatch(NUMBER_LINES);
    ArrayList<Integer> result = new ArrayList<Integer>();
    for (int i = 0; i < NUMBER_LINES; i++) {
      ByteBuffer buffer1 = AsynchronousFileImpl.newBuffer(SIZE);
      buffer1.rewind();
      for (int j = 0; j < SIZE; j++) {
        buffer1.put((byte) (j % Byte.MAX_VALUE));
      }
      CountDownCallback aio = new CountDownCallback(latch, null, result, i);
      controller.write(i * SIZE, SIZE, buffer1, aio);
    }

    // The buffer callback is only called after the complete callback was
    // called.
    // Because of that a race could happen on the assertions to
    // buffers.size what would invalidate the test
    // We close the file and that would guarantee the buffer callback was
    // called for all the elements
    controller.close();

    CountDownCallback.checkResults(NUMBER_LINES, result);

    // Make sure all the buffers are unique
    ByteBuffer lineOne = null;
    for (ByteBuffer bufferTmp : buffers) {
      if (lineOne == null) {
        lineOne = bufferTmp;
      } else {
        Assert.assertTrue(lineOne != bufferTmp);
      }
    }

    for (ByteBuffer bufferTmp : buffers) {
      destroy(bufferTmp);
    }

    buffers.clear();
  }
Exemplo n.º 2
0
  private void asyncData(final int numberOfLines, final int size, final int aioLimit)
      throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    controller.open(fileName, aioLimit);

    CountDownLatch latchDone = new CountDownLatch(numberOfLines);

    buffer = AsynchronousFileImpl.newBuffer(size);
    encodeBufer(buffer);

    preAlloc(controller, numberOfLines * size);

    ArrayList<CountDownCallback> list = new ArrayList<CountDownCallback>();

    ArrayList<Integer> result = new ArrayList<Integer>();

    for (int i = 0; i < numberOfLines; i++) {
      list.add(new CountDownCallback(latchDone, null, result, i));
    }

    long valueInitial = System.currentTimeMillis();

    long lastTime = System.currentTimeMillis();
    int counter = 0;
    for (CountDownCallback tmp : list) {
      controller.write(counter * size, size, buffer, tmp);
      if (++counter % 20000 == 0) {
        AsynchronousFileTest.debug(
            20000 * 1000 / (System.currentTimeMillis() - lastTime) + " rec/sec (Async)");
        lastTime = System.currentTimeMillis();
      }
    }

    UnitTestCase.waitForLatch(latchDone);

    long timeTotal = System.currentTimeMillis() - valueInitial;

    CountDownCallback.checkResults(numberOfLines, result);

    AsynchronousFileTest.debug(
        "After completions time = "
            + timeTotal
            + " for "
            + numberOfLines
            + " registers "
            + " size each line = "
            + size
            + ", Records/Sec="
            + numberOfLines * 1000 / timeTotal
            + " (Assynchronous)");

    for (CountDownCallback tmp : list) {
      Assert.assertEquals(1, tmp.timesDoneCalled.get());
      Assert.assertTrue(tmp.doneCalled);
      Assert.assertEquals(0, tmp.errorCalled);
    }

    controller.close();
  }
Exemplo n.º 3
0
 /**
  * Opening and closing a file immediately can lead to races on the native layer, creating crash
  * conditions.
  */
 @Test
 public void testOpenClose() throws Exception {
   controller = new AsynchronousFileImpl(executor, pollerExecutor);
   for (int i = 0; i < 1000; i++) {
     controller.open(fileName, 10000);
     controller.close();
   }
 }
  private void executeTest(final boolean sync) throws Throwable {
    MultiThreadAsynchronousFileTest.debug(sync ? "Sync test:" : "Async test");
    AsynchronousFileImpl jlibAIO = new AsynchronousFileImpl(executor, pollerExecutor);
    jlibAIO.open(FILE_NAME, 21000);
    try {
      MultiThreadAsynchronousFileTest.debug("Preallocating file");

      jlibAIO.fill(
          0l,
          MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS,
          MultiThreadAsynchronousFileTest.SIZE * MultiThreadAsynchronousFileTest.NUMBER_OF_LINES,
          (byte) 0);
      MultiThreadAsynchronousFileTest.debug("Done Preallocating file");

      CountDownLatch latchStart =
          new CountDownLatch(MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS + 1);

      ArrayList<ThreadProducer> list =
          new ArrayList<ThreadProducer>(MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS);
      for (int i = 0; i < MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS; i++) {
        ThreadProducer producer = new ThreadProducer("Thread " + i, latchStart, jlibAIO, sync);
        list.add(producer);
        producer.start();
      }

      latchStart.countDown();
      latchStart.await();

      long startTime = System.currentTimeMillis();

      for (ThreadProducer producer : list) {
        producer.join();
        if (producer.failed != null) {
          throw producer.failed;
        }
      }
      long endTime = System.currentTimeMillis();

      MultiThreadAsynchronousFileTest.debug(
          (sync ? "Sync result:" : "Async result:")
              + " Records/Second = "
              + MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS
                  * MultiThreadAsynchronousFileTest.NUMBER_OF_LINES
                  * 1000
                  / (endTime - startTime)
              + " total time = "
              + (endTime - startTime)
              + " total number of records = "
              + MultiThreadAsynchronousFileTest.NUMBER_OF_THREADS
                  * MultiThreadAsynchronousFileTest.NUMBER_OF_LINES);
    } finally {
      jlibAIO.close();
    }
  }
Exemplo n.º 5
0
  @Test
  public void testReleaseBuffers() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    WeakReference<ByteBuffer> bufferCheck = null;

    controller.open(fileName, 10000);
    bufferCheck = new WeakReference<ByteBuffer>(controller.getHandler());
    controller.fill(0, 10, 1024, (byte) 0);

    ByteBuffer write = AsynchronousFileImpl.newBuffer(1024);

    for (int i = 0; i < 1024; i++) {
      write.put(UnitTestCase.getSamplebyte(i));
    }

    final CountDownLatch latch = new CountDownLatch(1);

    controller.write(
        0,
        1024,
        write,
        new AIOCallback() {

          public void onError(final int errorCode, final String errorMessage) {}

          public void done() {
            latch.countDown();
          }
        });

    Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));

    WeakReference<ByteBuffer> bufferCheck2 = new WeakReference<ByteBuffer>(write);

    destroy(write);

    write = null;

    UnitTestCase.forceGC(bufferCheck2, 5000);

    Assert.assertNull(bufferCheck2.get());

    controller.close();

    controller = null;

    UnitTestCase.forceGC(bufferCheck, 5000);

    Assert.assertNull(bufferCheck.get());
  }
Exemplo n.º 6
0
 @Override
 @After
 public void tearDown() throws Exception {
   destroy(buffer);
   if (controller != null) {
     try {
       controller.close();
     } catch (Exception e) {
       // ignored
     }
   }
   executor.shutdown();
   pollerExecutor.shutdown();
   super.tearDown();
 }
Exemplo n.º 7
0
 @Test
 public void testFileNonExistent() throws Exception {
   controller = new AsynchronousFileImpl(executor, pollerExecutor);
   for (int i = 0; i < 1000; i++) {
     try {
       controller.open("/non-existent/IDontExist.error", 10000);
       Assert.fail(
           "Exception expected! The test could create a file called /non-existent/IDontExist.error when it was supposed to fail.");
     } catch (Exception ignored) {
     }
     try {
       controller.close();
       Assert.fail("Supposed to throw exception as the file wasn't opened");
     } catch (Exception ignored) {
     }
   }
 }
Exemplo n.º 8
0
  @Test
  public void testDirectSynchronous() throws Exception {

    final int NUMBER_LINES = 3000;
    final int SIZE = 1024;

    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    controller.open(fileName, 2000);

    buffer = AsynchronousFileImpl.newBuffer(SIZE);
    encodeBufer(buffer);

    preAlloc(controller, NUMBER_LINES * SIZE);

    long startTime = System.currentTimeMillis();

    for (int i = 0; i < NUMBER_LINES; i++) {
      CountDownLatch latchDone = new CountDownLatch(1);
      CountDownCallback aioBlock = new CountDownCallback(latchDone, null, null, 0);
      controller.write(i * 512, 512, buffer, aioBlock);
      UnitTestCase.waitForLatch(latchDone);
      Assert.assertTrue(aioBlock.doneCalled);
      Assert.assertEquals(0, aioBlock.errorCalled);
    }

    long timeTotal = System.currentTimeMillis() - startTime;
    AsynchronousFileTest.debug(
        "time = "
            + timeTotal
            + " for "
            + NUMBER_LINES
            + " registers "
            + " size each line = "
            + SIZE
            + " Records/Sec="
            + NUMBER_LINES * 1000 / timeTotal
            + " Synchronous");

    controller.close();
  }
Exemplo n.º 9
0
  /**
   * This test will call file.close() when there are still callbacks being processed. This could
   * cause a crash or callbacks missing and this test is validating both situations. The file is
   * also read after being written to validate its correctness
   */
  @Test
  public void testConcurrentClose() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    final int NUMBER_LINES = 1000;
    CountDownLatch readLatch = new CountDownLatch(NUMBER_LINES);
    final int SIZE = 1024;

    controller.open(fileName, 10000);

    controller.fill(0, 1, NUMBER_LINES * SIZE, (byte) 'j');

    controller.setBufferCallback(bufferCallbackDestroy);

    for (int i = 0; i < NUMBER_LINES; i++) {
      ByteBuffer buffer = AsynchronousFileImpl.newBuffer(SIZE);

      buffer.clear();
      addString("Str value " + i + "\n", buffer);
      for (int j = buffer.position(); j < buffer.capacity() - 1; j++) {
        buffer.put((byte) ' ');
      }
      buffer.put((byte) '\n');

      CountDownCallback aio = new CountDownCallback(readLatch, null, null, 0);
      controller.write(i * SIZE, SIZE, buffer, aio);
    }

    // If you call close you're supposed to wait events to finish before
    // closing it
    controller.close();

    controller.setBufferCallback(null);

    Assert.assertEquals(0, readLatch.getCount());
    waitForLatch(readLatch);
    controller.open(fileName, 10);

    ByteBuffer newBuffer = AsynchronousFileImpl.newBuffer(SIZE);

    ByteBuffer buffer = AsynchronousFileImpl.newBuffer(SIZE);

    for (int i = 0; i < NUMBER_LINES; i++) {
      newBuffer.clear();
      addString("Str value " + i + "\n", newBuffer);
      for (int j = newBuffer.position(); j < newBuffer.capacity() - 1; j++) {
        newBuffer.put((byte) ' ');
      }
      newBuffer.put((byte) '\n');

      CountDownLatch latch = new CountDownLatch(1);
      CountDownCallback aio = new CountDownCallback(latch, null, null, 0);
      controller.read(i * SIZE, SIZE, buffer, aio);
      waitForLatch(latch);
      Assert.assertEquals(0, aio.errorCalled);
      Assert.assertTrue(aio.doneCalled);

      byte bytesRead[] = new byte[SIZE];
      byte bytesCompare[] = new byte[SIZE];

      newBuffer.rewind();
      newBuffer.get(bytesCompare);
      buffer.rewind();
      buffer.get(bytesRead);

      for (int count = 0; count < SIZE; count++) {
        Assert.assertEquals(
            "byte position " + count + " differs on line " + i,
            bytesCompare[count],
            bytesRead[count]);
      }

      Assert.assertTrue(buffer.equals(newBuffer));
    }

    destroy(newBuffer);
  }
Exemplo n.º 10
0
  @Test
  public void testRead() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    controller.setBufferCallback(bufferCallbackDestroy);

    final int NUMBER_LINES = 1000;
    final int SIZE = 1024;

    controller.open(fileName, 1000);

    controller.fill(0, 1, NUMBER_LINES * SIZE, (byte) 'j');

    {
      CountDownLatch latch = new CountDownLatch(NUMBER_LINES);
      ArrayList<Integer> result = new ArrayList<Integer>();

      AtomicInteger errors = new AtomicInteger(0);

      for (int i = 0; i < NUMBER_LINES; i++) {
        if (i % 100 == 0) {
          System.out.println("Wrote " + i + " lines");
        }
        final ByteBuffer buffer0 = AsynchronousFileImpl.newBuffer(SIZE);
        for (int j = 0; j < SIZE; j++) {
          buffer0.put(UnitTestCase.getSamplebyte(j));
        }

        CountDownCallback aio = new CountDownCallback(latch, errors, result, i);
        controller.write(i * SIZE, SIZE, buffer0, aio);
      }

      waitForLatch(latch);

      Assert.assertEquals(0, errors.get());

      CountDownCallback.checkResults(NUMBER_LINES, result);
    }

    // If you call close you're supposed to wait events to finish before
    // closing it
    controller.close();
    controller.setBufferCallback(null);

    controller.open(fileName, 10);

    buffer = AsynchronousFileImpl.newBuffer(SIZE);

    for (int i = 0; i < NUMBER_LINES; i++) {
      if (i % 100 == 0) {
        System.out.println("Read " + i + " lines");
      }
      AsynchronousFileImpl.clearBuffer(buffer);

      CountDownLatch latch = new CountDownLatch(1);
      AtomicInteger errors = new AtomicInteger(0);
      CountDownCallback aio = new CountDownCallback(latch, errors, null, 0);

      controller.read(i * SIZE, SIZE, buffer, aio);

      waitForLatch(latch);
      Assert.assertEquals(0, errors.get());
      Assert.assertTrue(aio.doneCalled);

      byte bytesRead[] = new byte[SIZE];
      buffer.get(bytesRead);

      for (int count = 0; count < SIZE; count++) {
        Assert.assertEquals(
            "byte position " + count + " differs on line " + i + " position = " + count,
            UnitTestCase.getSamplebyte(count),
            bytesRead[count]);
      }
    }
  }
Exemplo n.º 11
0
  @Test
  public void testInvalidReads() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);

    final int SIZE = 512;

    controller.open(fileName, 10);
    controller.close();

    controller = new AsynchronousFileImpl(executor, pollerExecutor);

    controller.open(fileName, 10);

    controller.fill(0, 1, 512, (byte) 'j');

    buffer = AsynchronousFileImpl.newBuffer(SIZE);

    buffer.clear();

    for (int i = 0; i < SIZE; i++) {
      buffer.put((byte) (i % 100));
    }

    LocalCallback callbackLocal = new LocalCallback();

    controller.write(0, 512, buffer, callbackLocal);

    waitForLatch(callbackLocal.latch);

    {
      ByteBuffer newBuffer = AsynchronousFileImpl.newBuffer(512);

      try {
        callbackLocal = new LocalCallback();

        controller.read(0, 50, newBuffer, callbackLocal);

        waitForLatch(callbackLocal.latch);

        Assert.assertTrue(callbackLocal.error);

      } finally {
        // We have to destroy the native buffer manually as it was created with a malloc like C
        // function
        destroy(newBuffer);
        newBuffer = null;
      }
    }
    callbackLocal = new LocalCallback();

    byte bytes[] = new byte[512];

    {
      try {
        ByteBuffer newBuffer = ByteBuffer.wrap(bytes);

        controller.read(0, 512, newBuffer, callbackLocal);

        Assert.fail("An exception was supposed to be thrown");
      } catch (HornetQException ignored) {
        System.out.println(ignored);
      }
    }

    {
      final ByteBuffer newBuffer = AsynchronousFileImpl.newBuffer(512);
      try {
        callbackLocal = new LocalCallback();
        controller.read(0, 512, newBuffer, callbackLocal);
        waitForLatch(callbackLocal.latch);
        Assert.assertFalse(callbackLocal.error);

        newBuffer.rewind();

        byte[] bytesRead = new byte[SIZE];

        newBuffer.get(bytesRead);

        for (int i = 0; i < SIZE; i++) {
          Assert.assertEquals((byte) (i % 100), bytesRead[i]);
        }
      } finally {
        destroy(newBuffer);
      }
    }
  }
Exemplo n.º 12
0
  /**
   * This test is validating if the AIO layer can open two different simultaneous files without
   * loose any callbacks. This test made the native layer to crash at some point during development
   */
  @Test
  public void testTwoFiles() throws Exception {
    controller = new AsynchronousFileImpl(executor, pollerExecutor);
    final AsynchronousFileImpl controller2 = new AsynchronousFileImpl(executor, pollerExecutor);
    controller.open(fileName + ".1", 10000);
    controller2.open(fileName + ".2", 10000);

    int numberOfLines = 1000;
    int size = 1024;

    ArrayList<Integer> listResult1 = new ArrayList<Integer>();
    ArrayList<Integer> listResult2 = new ArrayList<Integer>();

    AtomicInteger errors = new AtomicInteger(0);

    try {
      CountDownLatch latchDone = new CountDownLatch(numberOfLines);
      CountDownLatch latchDone2 = new CountDownLatch(numberOfLines);

      buffer = AsynchronousFileImpl.newBuffer(size);
      encodeBufer(buffer);

      preAlloc(controller, numberOfLines * size);
      preAlloc(controller2, numberOfLines * size);

      ArrayList<CountDownCallback> list = new ArrayList<CountDownCallback>();
      ArrayList<CountDownCallback> list2 = new ArrayList<CountDownCallback>();

      for (int i = 0; i < numberOfLines; i++) {
        list.add(new CountDownCallback(latchDone, errors, listResult1, i));
        list2.add(new CountDownCallback(latchDone2, errors, listResult2, i));
      }

      int counter = 0;

      Iterator<CountDownCallback> iter2 = list2.iterator();

      for (CountDownCallback cb1 : list) {
        CountDownCallback cb2 = iter2.next();

        controller.write(counter * size, size, buffer, cb1);
        controller2.write(counter * size, size, buffer, cb2);
        ++counter;
      }

      UnitTestCase.waitForLatch(latchDone);
      UnitTestCase.waitForLatch(latchDone2);

      CountDownCallback.checkResults(numberOfLines, listResult1);
      CountDownCallback.checkResults(numberOfLines, listResult2);

      for (CountDownCallback callback : list) {
        Assert.assertEquals(1, callback.timesDoneCalled.get());
        Assert.assertTrue(callback.doneCalled);
      }

      for (CountDownCallback callback : list2) {
        Assert.assertEquals(1, callback.timesDoneCalled.get());
        Assert.assertTrue(callback.doneCalled);
      }

      Assert.assertEquals(0, errors.get());

      controller.close();
    } finally {
      try {
        controller2.close();
      } catch (Exception ignored) {
      }
    }
  }