Esempio n. 1
0
  public static Throwable throwable(@NotNull ValueIn valueIn, boolean appendCurrentStack) {
    Class type = valueIn.typePrefix();
    String preMessage = null;
    Throwable throwable = ObjectUtils.newInstance((Class<Throwable>) type);

    final String finalPreMessage = preMessage;
    final Throwable finalThrowable = throwable;
    final List<StackTraceElement> stes = new ArrayList<>();
    valueIn.marshallable(
        m -> {
          final String message = merge(finalPreMessage, m.read(() -> "message").text());

          if (message != null) {
            try {
              DETAILED_MESSAGE.set(finalThrowable, message);
            } catch (IllegalAccessException e) {
              throw Jvm.rethrow(e);
            }
          }
          m.read(() -> "stackTrace")
              .sequence(
                  stes,
                  (stes0, stackTrace) -> {
                    while (stackTrace.hasNextSequenceItem()) {
                      stackTrace.marshallable(
                          r -> {
                            final String declaringClass = r.read(() -> "class").text();
                            final String methodName = r.read(() -> "method").text();
                            final String fileName = r.read(() -> "file").text();
                            final int lineNumber = r.read(() -> "line").int32();

                            stes0.add(
                                new StackTraceElement(
                                    declaringClass, methodName, fileName, lineNumber));
                          });
                    }
                  });
        });

    if (appendCurrentStack) {
      stes.add(new StackTraceElement("~ remote", "tcp ~", "", 0));
      StackTraceElement[] stes2 = Thread.currentThread().getStackTrace();
      int first = 6;
      int last = Jvm.trimLast(first, stes2);
      //noinspection ManualArrayToCollectionCopy
      for (int i = first; i <= last; i++) stes.add(stes2[i]);
    }
    try {
      //noinspection ToArrayCallWithZeroLengthArrayArgument
      STACK_TRACE.set(finalThrowable, stes.toArray(NO_STE));
    } catch (IllegalAccessException e) {
      throw Jvm.rethrow(e);
    }
    return throwable;
  }
Esempio n. 2
0
  /**
   * Tests the performance of an event listener on the map for Insert events of 2 MB strings. Expect
   * it to handle at least 50 2 MB updates per second.
   */
  @Test
  public void testSubscriptionMapEventListenerInsertPerformance() {
    _testMap.clear();

    YamlLogging.setAll(true);
    // Create subscriber and register
    TestChronicleMapEventListener mapEventListener =
        new TestChronicleMapEventListener(_mapName, _twoMbTestStringLength);

    Subscriber<MapEvent> mapEventSubscriber = e -> e.apply(mapEventListener);
    clientAssetTree.registerSubscriber(_mapName, MapEvent.class, mapEventSubscriber);

    Jvm.pause(100);
    KVSSubscription subscription =
        (KVSSubscription) serverAssetTree.getAsset(_mapName).subscription(false);
    Assert.assertEquals(1, subscription.entrySubscriberCount());

    // Perform test a number of times to allow the JVM to warm up, but verify runtime against
    // average
    TestUtils.runMultipleTimesAndVerifyAvgRuntime(
        i -> {
          if (i > 0) {
            waitFor(() -> mapEventListener.getNoOfInsertEvents().get() >= _noOfPuts);
            Assert.assertEquals(_noOfPuts, mapEventListener.getNoOfInsertEvents().get());
          }
          // Test that the correct number of events were triggered on event listener
          Assert.assertEquals(0, mapEventListener.getNoOfRemoveEvents().get());
          Assert.assertEquals(0, mapEventListener.getNoOfUpdateEvents().get());

          _testMap.clear();

          mapEventListener.resetCounters();
        },
        () -> {
          IntStream.range(0, _noOfPuts)
              .forEach(
                  i -> {
                    _testMap.put(TestUtils.getKey(_mapName, i), _twoMbTestString);
                  });
        },
        _noOfRunsToAverage,
        2 * _secondInNanos);

    clientAssetTree.unregisterSubscriber(_mapName, mapEventSubscriber);

    Jvm.pause(1000);
    Assert.assertEquals(0, subscription.entrySubscriberCount());
  }
  @Override
  @ForceInline
  public byte readByte(long offset) {
    if (Jvm.isDebug()) checkReleased();

    return memory.readByte(address + translate(offset));
  }
Esempio n. 4
0
  /**
   * Tests the performance of an event listener on the map for Update events of 2 MB strings. Expect
   * it to handle at least 50 2 MB updates per second.
   */
  @Test
  public void testSubscriptionMapEventListenerUpdatePerformance() {
    _testMap.clear();

    // Put values before testing as we want to ignore the insert events
    Function<Integer, Object> putFunction =
        a -> _testMap.put(TestUtils.getKey(_mapName, a), _twoMbTestString);

    IntStream.range(0, _noOfPuts)
        .forEach(
            i -> {
              putFunction.apply(i);
            });

    Jvm.pause(100);
    // Create subscriber and register
    TestChronicleMapEventListener mapEventListener =
        new TestChronicleMapEventListener(_mapName, _twoMbTestStringLength);

    Subscriber<MapEvent> mapEventSubscriber = e -> e.apply(mapEventListener);
    clientAssetTree.registerSubscriber(
        _mapName + "?bootstrap=false", MapEvent.class, mapEventSubscriber);

    KVSSubscription subscription =
        (KVSSubscription) serverAssetTree.getAsset(_mapName).subscription(false);

    waitFor(() -> subscription.entrySubscriberCount() == 1);
    Assert.assertEquals(1, subscription.entrySubscriberCount());

    // Perform test a number of times to allow the JVM to warm up, but verify runtime against
    // average
    TestUtils.runMultipleTimesAndVerifyAvgRuntime(
        i -> {
          if (i > 0) {
            waitFor(() -> mapEventListener.getNoOfUpdateEvents().get() >= _noOfPuts);

            // Test that the correct number of events were triggered on event listener
            Assert.assertEquals(_noOfPuts, mapEventListener.getNoOfUpdateEvents().get());
          }
          Assert.assertEquals(0, mapEventListener.getNoOfInsertEvents().get());
          Assert.assertEquals(0, mapEventListener.getNoOfRemoveEvents().get());

          mapEventListener.resetCounters();
        },
        () -> {
          IntStream.range(0, _noOfPuts)
              .forEach(
                  i -> {
                    putFunction.apply(i);
                  });
        },
        _noOfRunsToAverage,
        3 * _secondInNanos);
    clientAssetTree.unregisterSubscriber(_mapName, mapEventSubscriber);

    waitFor(() -> subscription.entrySubscriberCount() == 0);
    Assert.assertEquals(0, subscription.entrySubscriberCount());
  }
Esempio n. 5
0
  /**
   * Test that listening to events for a given key can handle 50 updates per second of 2 MB string
   * values.
   */
  @Test
  public void testSubscriptionMapEventOnKeyPerformance() {
    _testMap.clear();

    String key = TestUtils.getKey(_mapName, 0);

    // Create subscriber and register
    // Add 4 for the number of puts that is added to the string
    TestChronicleKeyEventSubscriber keyEventSubscriber =
        new TestChronicleKeyEventSubscriber(_twoMbTestStringLength);

    clientAssetTree.registerSubscriber(
        _mapName + "/" + key + "?bootstrap=false", String.class, keyEventSubscriber);
    Jvm.pause(100);
    Asset child = serverAssetTree.getAsset(_mapName).getChild(key);
    Assert.assertNotNull(child);
    Subscription subscription = child.subscription(false);
    Assert.assertEquals(1, subscription.subscriberCount());

    long start = System.nanoTime();
    // Perform test a number of times to allow the JVM to warm up, but verify runtime against
    // average
    TestUtils.runMultipleTimesAndVerifyAvgRuntime(
        () -> {
          IntStream.range(0, _noOfPuts)
              .forEach(
                  i -> {
                    _testMap.put(key, _twoMbTestString);
                  });
        },
        _noOfRunsToAverage,
        3 * _secondInNanos);

    waitFor(() -> keyEventSubscriber.getNoOfEvents().get() >= _noOfPuts * _noOfRunsToAverage);
    long time = System.nanoTime() - start;
    System.out.printf("Took %.3f seconds to receive all events%n", time / 1e9);

    // Test that the correct number of events was triggered on event listener
    Assert.assertEquals(_noOfPuts * _noOfRunsToAverage, keyEventSubscriber.getNoOfEvents().get());

    clientAssetTree.unregisterSubscriber(_mapName + "/" + key, keyEventSubscriber);

    Jvm.pause(100);
    Assert.assertEquals(0, subscription.subscriberCount());
  }
 public boolean equalsBytes(@NotNull Bytes b2, long remaining) {
   long i = 0;
   try {
     for (; i < remaining - 7; i++)
       if (readLong(readPosition() + i) != b2.readLong(b2.readPosition() + i)) return false;
     for (; i < remaining; i++)
       if (readByte(readPosition() + i) != b2.readByte(b2.readPosition() + i)) return false;
   } catch (BufferUnderflowException | IORuntimeException e) {
     throw Jvm.rethrow(e);
   }
   return true;
 }
Esempio n. 7
0
  /**
   * Test that listening to events for a given map can handle 50 updates per second of 2 MB string
   * values and are triggering events which contain both the key and value (topic).
   */
  @Test
  public void testSubscriptionMapEventOnTopicPerformance() {
    _testMap.clear();

    String key = TestUtils.getKey(_mapName, 0);

    // Create subscriber and register
    TestChronicleTopicSubscriber topicSubscriber =
        new TestChronicleTopicSubscriber(key, _twoMbTestStringLength);

    clientAssetTree.registerTopicSubscriber(_mapName, String.class, String.class, topicSubscriber);

    Jvm.pause(100);
    KVSSubscription subscription =
        (KVSSubscription) serverAssetTree.getAsset(_mapName).subscription(false);
    Assert.assertEquals(1, subscription.topicSubscriberCount());

    // Perform test a number of times to allow the JVM to warm up, but verify runtime against
    // average
    TestUtils.runMultipleTimesAndVerifyAvgRuntime(
        i -> {
          System.out.println("test");
          int events = _noOfPuts * i;
          waitFor(() -> events == topicSubscriber.getNoOfEvents().get());
          Assert.assertEquals(events, topicSubscriber.getNoOfEvents().get());
        },
        () -> {
          IntStream.range(0, _noOfPuts)
              .forEach(
                  i -> {
                    _testMap.put(key, _twoMbTestString);
                  });
        },
        _noOfRunsToAverage,
        3 * _secondInNanos);

    // Test that the correct number of events was triggered on event listener
    int events = _noOfPuts * _noOfRunsToAverage;
    waitFor(() -> events == topicSubscriber.getNoOfEvents().get());
    Assert.assertEquals(events, topicSubscriber.getNoOfEvents().get());

    clientAssetTree.unregisterTopicSubscriber(_mapName, topicSubscriber);
    waitFor(() -> 0 == subscription.topicSubscriberCount());
    Assert.assertEquals(0, subscription.topicSubscriberCount());
  }
 @Override
 public void run() {
   while (running && !Thread.currentThread().isInterrupted()) {
     Jvm.pause(1);
     long delay = System.nanoTime() - sample;
     if (delay > 1000 * 1000) {
       System.out.println(
           "\n"
               + (System.currentTimeMillis() - START_TIME)
               + " : Delay of "
               + delay / 100000 / 10.0
               + " ms.");
       int count = 0;
       for (StackTraceElement ste : thread.getStackTrace()) {
         System.out.println("\tat " + ste);
         if (count++ > 6) break;
       }
     }
   }
 }
  /**
   * because we have set System.setProperty("Throttler.maxEventsPreSecond", "1"); in the static
   * above, we will only get one event per second, this test also checks that the messages still
   * arrive in order.
   *
   * @throws IOException
   * @throws InterruptedException
   */
  @Test
  public void testReceivingThrottledEventsInOrder() throws IOException, InterruptedException {

    final BlockingQueue<String> eventsQueue = new LinkedBlockingDeque<>();

    YamlLogging.showServerWrites = true;
    YamlLogging.showServerReads = true;

    yamlLoggger(
        () -> {
          try {

            Subscriber<String> add = eventsQueue::add;
            assetTree.registerSubscriber(NAME, String.class, add);

            for (int i = 0; i < 10; i++) {
              map.put("Hello" + i, "World" + i);
            }

            final long start = System.currentTimeMillis();

            for (int i = 0; i < 10; i++) {
              String actual = eventsQueue.poll(5, SECONDS);
              Assert.assertNotNull(actual);
              Assert.assertEquals("Hello" + i, actual);
            }

            // because we are only sending 1 message per second this should take around 10
            // seconds, certainly longer than 5 seconds
            Assert.assertTrue(System.currentTimeMillis() > start + TimeUnit.SECONDS.toMillis(5));

          } catch (Exception e) {
            throw Jvm.rethrow(e);
          }
        });
  }
 @Override
 public void queryThrown(Throwable t, String query, Object... args) {
   throw Jvm.rethrow(t);
 }
  @NotNull
  private ReadMarshallable wireInConsumer() {
    return (wire) -> {
      long startWritePosition = outWire.bytes().writePosition();

      // if true the next data message will be a system message
      isSystemMessage = wire.bytes().readRemaining() == 0;
      if (isSystemMessage) {
        if (LOG.isDebugEnabled()) LOG.debug("received system-meta-data");
        return;
      }

      try {
        readCsp(wire);
        readTid(wire);
      } catch (Throwable t) {
        Jvm.rethrow(t);
      }

      try {
        if (hasCspChanged(cspText)) {

          if (LOG.isDebugEnabled()) LOG.debug("received meta-data:\n" + wire.bytes().toHexString());

          requestContext = requestContextInterner.intern(cspText);
          viewType = requestContext.viewType();
          if (viewType == null) {
            if (LOG.isDebugEnabled()) LOG.debug("received system-meta-data");
            isSystemMessage = true;
            return;
          }

          view = this.assetTree.acquireView(requestContext);

          if (viewType == MapView.class
              || viewType == EntrySetView.class
              || viewType == ValuesCollection.class
              || viewType == KeySetView.class
              || viewType == ObjectKVSSubscription.class
              || viewType == TopicPublisher.class
              || viewType == Publisher.class
              || viewType == Reference.class
              || viewType == TopologySubscription.class
              || viewType == Replication.class
              || viewType == Heartbeat.class) {

            // default to string type if not provided
            final Class type =
                requestContext.keyType() == null ? String.class : requestContext.keyType();

            final Class type2 =
                requestContext.valueType() == null ? String.class : requestContext.valueType();

            wireAdapter = new GenericWireAdapter(type, type2);
          } else throw new UnsupportedOperationException("unsupported view type");
        }
      } catch (Throwable e) {
        LOG.error("", e);
        outWire.bytes().writePosition(startWritePosition);
        outWire.writeDocument(true, w -> w.writeEventName(CoreFields.tid).int64(tid));
        outWire.writeDocument(false, out -> out.writeEventName(() -> "exception").throwable(e));
        logYamlToStandardOut(outWire);
        rethrow(e);
      }
    };
  }
Esempio n. 12
0
@SuppressWarnings("sunapi")
public class NativeBytesStore<Underlying>
    implements BytesStore<NativeBytesStore<Underlying>, Underlying> {
  private static final long MEMORY_MAPPED_SIZE = 128 << 10;
  private static final Logger LOGGER = LoggerFactory.getLogger(NativeBytesStore.class);

  @Nullable private final Cleaner cleaner;
  private final boolean elastic;
  @Nullable private final Underlying underlyingObject;

  @Nullable
  private final Throwable createdHere = Jvm.isDebug() ? new Throwable("Created here") : null;
  // on release, set this to null.
  @Nullable protected Memory memory = OS.memory();
  private final ReferenceCounter refCount = ReferenceCounter.onReleased(this::performRelease);
  protected long address;
  long maximumLimit;
  private Error releasedHere;

  private NativeBytesStore(@NotNull ByteBuffer bb, boolean elastic) {
    this.elastic = elastic;
    underlyingObject = (Underlying) bb;
    setAddress(((DirectBuffer) bb).address());
    this.maximumLimit = bb.capacity();
    cleaner = ((DirectBuffer) bb).cleaner();
  }

  public NativeBytesStore(long address, long maximumLimit) {
    this(address, maximumLimit, null, false);
  }

  public NativeBytesStore(
      long address, long maximumLimit, @Nullable Runnable deallocator, boolean elastic) {
    setAddress(address);
    this.maximumLimit = maximumLimit;
    cleaner = deallocator == null ? null : Cleaner.create(this, deallocator);
    underlyingObject = null;
    this.elastic = elastic;
  }

  @NotNull
  public static NativeBytesStore<ByteBuffer> wrap(@NotNull ByteBuffer bb) {
    return new NativeBytesStore<>(bb, false);
  }

  /**
   * this is an elastic native store
   *
   * @param capacity of the buffer.
   */
  @NotNull
  public static NativeBytesStore<Void> nativeStore(long capacity) throws IllegalArgumentException {
    return of(capacity, true, true);
  }

  @NotNull
  private static NativeBytesStore<Void> of(long capacity, boolean zeroOut, boolean elastic)
      throws IllegalArgumentException {
    Memory memory = OS.memory();
    long address = memory.allocate(capacity);
    if (zeroOut || capacity < MEMORY_MAPPED_SIZE) {
      memory.setMemory(address, capacity, (byte) 0);
      memory.storeFence();
    }
    Deallocator deallocator = new Deallocator(address, capacity);
    return new NativeBytesStore<>(address, capacity, deallocator, elastic);
  }

  @NotNull
  public static NativeBytesStore<Void> nativeStoreWithFixedCapacity(long capacity)
      throws IllegalArgumentException {
    return of(capacity, true, false);
  }

  @NotNull
  public static NativeBytesStore<Void> lazyNativeBytesStoreWithFixedCapacity(long capacity)
      throws IllegalArgumentException {
    return of(capacity, false, false);
  }

  @NotNull
  public static NativeBytesStore<ByteBuffer> elasticByteBuffer() {
    return elasticByteBuffer(OS.pageSize());
  }

  @NotNull
  public static NativeBytesStore<ByteBuffer> elasticByteBuffer(int size) {
    return new NativeBytesStore<>(ByteBuffer.allocateDirect(size), true);
  }

  @NotNull
  @Override
  public BytesStore<NativeBytesStore<Underlying>, Underlying> copy() throws IllegalStateException {
    if (underlyingObject == null) {
      NativeBytesStore<Void> copy = of(realCapacity(), false, true);
      OS.memory().copyMemory(address, copy.address, capacity());
      return (BytesStore) copy;

    } else if (underlyingObject instanceof ByteBuffer) {
      ByteBuffer bb = ByteBuffer.allocateDirect(Maths.toInt32(capacity()));
      bb.put((ByteBuffer) underlyingObject);
      bb.clear();
      return (BytesStore) wrap(bb);

    } else {
      throw new UnsupportedOperationException();
    }
  }

  @Override
  public VanillaBytes<Underlying> bytesForWrite() throws IllegalStateException {
    return elastic ? new NativeBytes<>(this) : new VanillaBytes<>(this);
  }

  @Override
  @ForceInline
  public long realCapacity() {
    return maximumLimit;
  }

  @Override
  @ForceInline
  public long capacity() {
    return maximumLimit;
  }

  @Nullable
  @Override
  @ForceInline
  public Underlying underlyingObject() {
    return underlyingObject;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> zeroOut(long start, long end)
      throws IllegalArgumentException {
    if (start < writePosition() || end > writeLimit())
      throw new IllegalArgumentException(
          "position: "
              + writePosition()
              + ", start: "
              + start
              + ", end: "
              + end
              + ", limit: "
              + writeLimit());
    if (start >= end) return this;

    memory.setMemory(address + translate(start), end - start, (byte) 0);
    return this;
  }

  @Override
  @ForceInline
  public boolean compareAndSwapInt(long offset, int expected, int value) {
    return memory.compareAndSwapInt(address + translate(offset), expected, value);
  }

  @Override
  @ForceInline
  public boolean compareAndSwapLong(long offset, long expected, long value) {
    return memory.compareAndSwapLong(address + translate(offset), expected, value);
  }

  long translate(long offset) {
    long offset2 = offset - start();
    //        assert checkTranslatedBounds(offset2);
    return offset2;
  }

  public long start() {
    return 0L;
  }

  @Override
  public void reserve() throws IllegalStateException {
    refCount.reserve();
  }

  @Override
  public void release() throws IllegalStateException {
    refCount.release();
    if (Jvm.isDebug() && refCount.get() == 0) releasedHere = new Error("Released here");
  }

  @Override
  public long refCount() {
    return refCount.get();
  }

  @Override
  @ForceInline
  public byte readByte(long offset) {
    if (Jvm.isDebug()) checkReleased();

    return memory.readByte(address + translate(offset));
  }

  public void checkReleased() {
    if (releasedHere != null)
      throw new InternalError("Accessing a released resource", releasedHere);
  }

  @Override
  @ForceInline
  public short readShort(long offset) {
    return memory.readShort(address + translate(offset));
  }

  @Override
  @ForceInline
  public int readInt(long offset) {
    return memory.readInt(address + translate(offset));
  }

  @Override
  @ForceInline
  public long readLong(long offset) {
    return memory.readLong(address + translate(offset));
  }

  @Override
  @ForceInline
  public float readFloat(long offset) {
    return memory.readFloat(address + translate(offset));
  }

  @Override
  @ForceInline
  public double readDouble(long offset) {
    return memory.readDouble(address + translate(offset));
  }

  @Override
  @ForceInline
  public byte readVolatileByte(long offset) {
    return memory.readVolatileByte(address + translate(offset));
  }

  @Override
  @ForceInline
  public short readVolatileShort(long offset) {
    return memory.readVolatileShort(address + translate(offset));
  }

  @Override
  @ForceInline
  public int readVolatileInt(long offset) {
    return memory.readVolatileInt(address + translate(offset));
  }

  @Override
  @ForceInline
  public long readVolatileLong(long offset) {
    return memory.readVolatileLong(address + translate(offset));
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeByte(long offset, byte i8) {
    memory.writeByte(address + translate(offset), i8);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeShort(long offset, short i16) {
    memory.writeShort(address + translate(offset), i16);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeInt(long offset, int i32) {
    memory.writeInt(address + translate(offset), i32);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeOrderedInt(long offset, int i) {
    memory.writeOrderedInt(address + translate(offset), i);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeLong(long offset, long i64) {
    memory.writeLong(address + translate(offset), i64);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeOrderedLong(long offset, long i) {
    memory.writeOrderedLong(address + translate(offset), i);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeFloat(long offset, float f) {
    memory.writeFloat(address + translate(offset), f);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeDouble(long offset, double d) {
    memory.writeDouble(address + translate(offset), d);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeVolatileByte(long offset, byte i8) {
    memory.writeVolatileByte(address + translate(offset), i8);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeVolatileShort(long offset, short i16) {
    memory.writeVolatileShort(address + translate(offset), i16);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeVolatileInt(long offset, int i32) {
    memory.writeVolatileInt(address + translate(offset), i32);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> writeVolatileLong(long offset, long i64) {
    memory.writeVolatileLong(address + translate(offset), i64);
    return this;
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> write(
      long offsetInRDO, byte[] bytes, int offset, int length) {
    memory.copyMemory(bytes, offset, address + translate(offsetInRDO), length);
    return this;
  }

  @Override
  @ForceInline
  public void write(long offsetInRDO, @NotNull ByteBuffer bytes, int offset, int length) {
    if (bytes.isDirect()) {
      memory.copyMemory(((DirectBuffer) bytes).address(), address + translate(offsetInRDO), length);

    } else {
      memory.copyMemory(bytes.array(), offset, address + translate(offsetInRDO), length);
    }
  }

  @NotNull
  @Override
  @ForceInline
  public NativeBytesStore<Underlying> write(
      long offsetInRDO, @NotNull RandomDataInput bytes, long offset, long length)
      throws BufferOverflowException, BufferUnderflowException, IORuntimeException {
    // TODO optimize, call unsafe.copyMemory when possible, copy 4, 2 bytes at once
    long i = 0;
    for (; i < length - 7; i += 8) {
      writeLong(offsetInRDO + i, bytes.readLong(offset + i));
    }
    for (; i < length; i++) {
      writeByte(offsetInRDO + i, bytes.readByte(offset + i));
    }
    return this;
  }

  @Override
  public long address(long offset) throws UnsupportedOperationException {
    if (offset < start() || offset >= capacity()) throw new IllegalArgumentException();
    return address + translate(offset);
  }

  private void performRelease() {
    if (refCount.get() > 0) {
      LOGGER.info("NativeBytesStore discarded without releasing ", createdHere);
    }

    memory = null;
    if (cleaner != null) cleaner.clean();
  }

  @NotNull
  @Override
  public String toString() {
    try {
      return BytesInternal.toString(this);
    } catch (IllegalStateException | IORuntimeException e) {
      return e.toString();
    }
  }

  @Override
  @ForceInline
  public void nativeRead(long position, long address, long size) {
    // TODO add bounds checking.
    OS.memory().copyMemory(address(position), address, size);
  }

  @Override
  @ForceInline
  public void nativeWrite(long address, long position, long size) {
    // TODO add bounds checking.
    this.memory.copyMemory(address, address(position), size);
  }

  void write8bit(long position, char[] chars, int offset, int length) {
    long addr = address + translate(position);
    Memory memory = this.memory;
    for (int i = 0; i < length; i++) memory.writeByte(addr + i, (byte) chars[offset + i]);
  }

  void read8bit(long position, char[] chars, int length) {
    long addr = address + translate(position);
    Memory memory = OS.memory();
    for (int i = 0; i < length; i++) chars[i] = (char) (memory.readByte(addr + i) & 0xFF);
  }

  public long readIncompleteLong(long offset) {
    int remaining = (int) Math.min(8, readRemaining() - offset);
    long l = 0;
    for (int i = 0; i < remaining; i++) {
      byte b = memory.readByte(address + offset + i);
      l |= (long) (b & 0xFF) << (i * 8);
    }
    return l;
  }

  @Override
  public boolean equals(Object obj) {
    try {
      return obj instanceof BytesStore && BytesInternal.contentEqual(this, (BytesStore) obj);
    } catch (IORuntimeException e) {
      throw new AssertionError(e);
    }
  }

  public void setAddress(long address) {
    if ((address & ~0x3FFF) == 0)
      throw new AssertionError("Invalid address " + Long.toHexString(address));
    this.address = address;
  }

  public long appendUTF(long pos, char[] chars, int offset, int length) {
    long address = this.address + translate(0);
    Memory memory = this.memory;
    int i;
    ascii:
    {
      for (i = 0; i < length; i++) {
        char c = chars[offset + i];
        if (c > 0x007F) break ascii;
        memory.writeByte(address + pos++, (byte) c);
      }

      return pos;
    }
    return appendUTF0(pos, chars, offset, length, i);
  }

  private long appendUTF0(long pos, char[] chars, int offset, int length, int i) {
    for (; i < length; i++) {
      char c = chars[offset + i];
      if (c <= 0x007F) {
        writeByte(pos++, (byte) c);

      } else if (c <= 0x07FF) {
        writeByte(pos++, (byte) (0xC0 | ((c >> 6) & 0x1F)));
        writeByte(pos++, (byte) (0x80 | c & 0x3F));

      } else {
        writeByte(pos++, (byte) (0xE0 | ((c >> 12) & 0x0F)));
        writeByte(pos++, (byte) (0x80 | ((c >> 6) & 0x3F)));
        writeByte(pos++, (byte) (0x80 | (c & 0x3F)));
      }
    }
    return pos;
  }

  static class Deallocator implements Runnable {
    private volatile long address, size;

    Deallocator(long address, long size) {
      assert address != 0;
      this.address = address;
      this.size = size;
    }

    @Override
    public void run() {
      if (address == 0) return;
      address = 0;
      OS.memory().freeMemory(address, size);
    }
  }
}
Esempio n. 13
0
 @Override
 public void release() throws IllegalStateException {
   refCount.release();
   if (Jvm.isDebug() && refCount.get() == 0) releasedHere = new Error("Released here");
 }
Esempio n. 14
0
 public static void reset() {
   DESC_TO_SERVER_SOCKET_CHANNEL_MAP.values().forEach(Closeable::closeQuietly);
   HOSTNAME_PORT_ALIAS.clear();
   DESC_TO_SERVER_SOCKET_CHANNEL_MAP.clear();
   Jvm.pause(500);
 }
Esempio n. 15
0
 private void waitFor(BooleanSupplier b) {
   for (int i = 1; i <= 40; i++) if (!b.getAsBoolean()) Jvm.pause(i * i);
 }
Esempio n. 16
0
/** Created by peter.lawrey on 16/01/15. */
enum WireInternal {
  ;
  public static final StringInterner INTERNER = new StringInterner(128);
  static final StringBuilderPool SBP = new StringBuilderPool();
  static final StringBuilderPool ASBP = new StringBuilderPool();
  static final StackTraceElement[] NO_STE = {};
  private static final Field DETAILED_MESSAGE = Jvm.getField(Throwable.class, "detailMessage");
  private static final Field STACK_TRACE = Jvm.getField(Throwable.class, "stackTrace");

  static {
    ClassAliasPool.CLASS_ALIASES.addAlias(WireSerializedLambda.class, "SerializedLambda");
    ClassAliasPool.CLASS_ALIASES.addAlias(WireType.class);
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializableFunction.class, "Function");
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializableBiFunction.class, "BiFunction");
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializableConsumer.class, "Consumer");
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializablePredicate.class, "Predicate");
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializableUpdater.class, "Updater");
    ClassAliasPool.CLASS_ALIASES.addAlias(SerializableUpdaterWithArg.class, "UpdaterWithArg");
  }

  public static <E extends Enum<E>> E internEnum(Class<E> eClass, CharSequence cs) {
    return (E) EnumInterner.ENUM_INTERNER.get(eClass).intern(cs);
  }

  public static StringBuilder acquireStringBuilder() {
    return SBP.acquireStringBuilder();
  }

  public static StringBuilder acquireAnotherStringBuilder(CharSequence cs) {
    StringBuilder sb = ASBP.acquireStringBuilder();
    assert sb != cs;
    return sb;
  }

  public static void writeData(
      @NotNull WireOut wireOut,
      boolean metaData,
      boolean notReady,
      @NotNull WriteMarshallable writer) {
    Bytes bytes = wireOut.bytes();
    long position = bytes.writePosition();
    int metaDataBit = metaData ? Wires.META_DATA : 0;
    bytes.writeOrderedInt(metaDataBit | Wires.NOT_READY | Wires.UNKNOWN_LENGTH);
    writer.writeMarshallable(wireOut);
    int length =
        metaDataBit
            | toIntU30(
                bytes.writePosition() - position - 4,
                "Document length %,d out of 30-bit int range.");
    bytes.writeOrderedInt(position, length | (notReady ? Wires.NOT_READY : 0));
  }

  public static void writeDataOnce(
      @NotNull WireOut wireOut, boolean metaData, @NotNull WriteMarshallable writer) {
    Bytes bytes = wireOut.bytes();
    long position = bytes.writePosition();
    int metaDataBit = metaData ? Wires.META_DATA : 0;
    int value = metaDataBit | Wires.NOT_READY | Wires.UNKNOWN_LENGTH;
    if (!bytes.compareAndSwapInt(position, 0, value)) return;
    bytes.writeSkip(4);
    writer.writeMarshallable(wireOut);
    int length =
        metaDataBit
            | toIntU30(
                bytes.writePosition() - position - 4,
                "Document length %,d out of 30-bit int range.");
    if (!bytes.compareAndSwapInt(position, value, length | Wires.META_DATA))
      throw new AssertionError();
  }

  public static boolean readData(
      long offset,
      @NotNull WireIn wireIn,
      @Nullable ReadMarshallable metaDataConsumer,
      @Nullable ReadMarshallable dataConsumer) {
    final Bytes bytes = wireIn.bytes();

    long position = bytes.readPosition();
    long limit = bytes.readLimit();
    try {
      bytes.readLimit(bytes.isElastic() ? bytes.capacity() : bytes.realCapacity());
      bytes.readPosition(offset);
      return readData(wireIn, metaDataConsumer, dataConsumer);
    } finally {
      bytes.readLimit(limit);
      bytes.readPosition(position);
    }
  }

  public static boolean readData(
      @NotNull WireIn wireIn,
      @Nullable ReadMarshallable metaDataConsumer,
      @Nullable ReadMarshallable dataConsumer) {
    final Bytes<?> bytes = wireIn.bytes();
    boolean read = false;
    while (bytes.readRemaining() >= 4) {
      long position = bytes.readPosition();
      int header = bytes.readVolatileInt(position);
      if (!isKnownLength(header)) return read;
      bytes.readSkip(4);
      final boolean ready = Wires.isReady(header);
      final int len = Wires.lengthOf(header);
      if (Wires.isData(header)) {
        if (dataConsumer == null) {
          return false;

        } else {
          ((InternalWireIn) wireIn).setReady(ready);
          bytes.readWithLength(len, b -> dataConsumer.readMarshallable(wireIn));
          return true;
        }
      } else {

        if (metaDataConsumer == null) {
          // skip the header
          bytes.readSkip(len);
        } else {
          // bytes.readWithLength(len, b -> metaDataConsumer.accept(wireIn));
          // inlined to avoid garbage
          if ((long) len > bytes.readRemaining()) throw new BufferUnderflowException();
          long limit0 = bytes.readLimit();
          long limit = bytes.readPosition() + (long) len;
          try {
            bytes.readLimit(limit);
            metaDataConsumer.readMarshallable(wireIn);
          } finally {
            bytes.readLimit(limit0);
            bytes.readPosition(limit);
          }
        }

        if (dataConsumer == null) return true;
        read = true;
      }
    }
    return read;
  }

  public static void rawReadData(@NotNull WireIn wireIn, @NotNull ReadMarshallable dataConsumer) {
    final Bytes<?> bytes = wireIn.bytes();
    int header = bytes.readInt();
    assert Wires.isReady(header) && Wires.isData(header);
    final int len = Wires.lengthOf(header);

    long limit0 = bytes.readLimit();
    long limit = bytes.readPosition() + (long) len;
    try {
      bytes.readLimit(limit);
      dataConsumer.readMarshallable(wireIn);
    } finally {
      bytes.readLimit(limit0);
    }
  }

  @Deprecated
  public static String fromSizePrefixedBinaryToText(@NotNull Bytes bytes) {
    return Wires.fromSizePrefixedBlobs(bytes);
  }

  @NotNull
  static String fromSizePrefixedBlobs(@NotNull Bytes bytes, long position, long length) {
    StringBuilder sb = new StringBuilder();

    final long limit0 = bytes.readLimit();
    final long position0 = bytes.readPosition();
    try {
      bytes.readPosition(position);
      long limit2 = Math.min(limit0, position + length);
      bytes.readLimit(limit2);
      long missing = position + length - limit2;
      while (bytes.readRemaining() >= 4) {
        long header = bytes.readUnsignedInt();
        int len = Wires.lengthOf(header);
        if (len > bytes.readRemaining())
          throw new RuntimeException(
              "Are you sure this was written with writeDocument and has a 4 byte size prefix, "
                  + len
                  + " > "
                  + bytes.readRemaining());
        String type =
            Wires.isData(header)
                ? Wires.isReady(header) ? "!!data" : "!!not-ready-data!"
                : Wires.isReady(header) ? "!!meta-data" : "!!not-ready-meta-data!";
        boolean binary = bytes.readByte(bytes.readPosition()) < ' ';

        sb.append("--- ").append(type).append(binary ? " #binary" : "");
        if (missing > 0) sb.append(" # missing: ").append(missing);
        if (len > bytes.readRemaining())
          sb.append(" # len: ").append(len).append(", remaining: ").append(bytes.readRemaining());
        sb.append("\n");

        Bytes textBytes = bytes;

        if (binary) {
          Bytes bytes2 = Bytes.elasticByteBuffer();
          TextWire textWire = new TextWire(bytes2);
          long readLimit = bytes.readLimit();

          try {
            bytes.readLimit(bytes.readPosition() + len);
            new BinaryWire(bytes).copyTo(textWire);
          } finally {
            bytes.readLimit(readLimit);
          }
          textBytes = bytes2;
          len = (int) textBytes.readRemaining();
        }
        try {
          for (int i = 0; i < len; i++) {
            int ch = textBytes.readUnsignedByte();
            sb.append((char) ch);
          }
        } catch (Exception e) {
          sb.append(" ").append(e);
        }
        if (sb.charAt(sb.length() - 1) != '\n') sb.append('\n');
      }

      return sb.toString();
    } finally {
      bytes.readLimit(limit0);
      bytes.readPosition(position0);
    }
  }

  private static boolean isKnownLength(long len) {
    return (len & (Wires.META_DATA | Wires.LENGTH_MASK)) != Wires.UNKNOWN_LENGTH;
  }

  public static Throwable throwable(@NotNull ValueIn valueIn, boolean appendCurrentStack) {
    Class type = valueIn.typePrefix();
    String preMessage = null;
    Throwable throwable = ObjectUtils.newInstance((Class<Throwable>) type);

    final String finalPreMessage = preMessage;
    final Throwable finalThrowable = throwable;
    final List<StackTraceElement> stes = new ArrayList<>();
    valueIn.marshallable(
        m -> {
          final String message = merge(finalPreMessage, m.read(() -> "message").text());

          if (message != null) {
            try {
              DETAILED_MESSAGE.set(finalThrowable, message);
            } catch (IllegalAccessException e) {
              throw Jvm.rethrow(e);
            }
          }
          m.read(() -> "stackTrace")
              .sequence(
                  stes,
                  (stes0, stackTrace) -> {
                    while (stackTrace.hasNextSequenceItem()) {
                      stackTrace.marshallable(
                          r -> {
                            final String declaringClass = r.read(() -> "class").text();
                            final String methodName = r.read(() -> "method").text();
                            final String fileName = r.read(() -> "file").text();
                            final int lineNumber = r.read(() -> "line").int32();

                            stes0.add(
                                new StackTraceElement(
                                    declaringClass, methodName, fileName, lineNumber));
                          });
                    }
                  });
        });

    if (appendCurrentStack) {
      stes.add(new StackTraceElement("~ remote", "tcp ~", "", 0));
      StackTraceElement[] stes2 = Thread.currentThread().getStackTrace();
      int first = 6;
      int last = Jvm.trimLast(first, stes2);
      //noinspection ManualArrayToCollectionCopy
      for (int i = first; i <= last; i++) stes.add(stes2[i]);
    }
    try {
      //noinspection ToArrayCallWithZeroLengthArrayArgument
      STACK_TRACE.set(finalThrowable, stes.toArray(NO_STE));
    } catch (IllegalAccessException e) {
      throw Jvm.rethrow(e);
    }
    return throwable;
  }

  @Nullable
  static String merge(@Nullable String a, @Nullable String b) {
    return a == null ? b : b == null ? a : a + " " + b;
  }

  public static void compress(ValueOut out, String compression, Bytes bytes) {
    switch (compression) {
      case "snappy":
        try {
          out.bytes("snappy", Snappy.compress(bytes.toByteArray()));
        } catch (IOException e) {
          throw new AssertionError(e);
        }
        break;
      case "gzip":
        try {
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          GZIPOutputStream gos = new GZIPOutputStream(baos);
          bytes.copyTo(gos);
          gos.close();
          out.bytes("gzip", baos.toByteArray());
        } catch (IOException e) {
          throw new AssertionError(e);
        }
        break;
      default:
        throw new IllegalArgumentException("Unknown compression " + compression);
    }
  }

  @Deprecated
  public static void compress(ValueOut out, String compression, String str) {
    if (compression.equals("snappy")) {
      try {
        out.bytes("snappy", Snappy.compress(str));
      } catch (IOException e) {
        throw new AssertionError(e);
      }
    } else {
      throw new IllegalArgumentException("Unknown compression " + compression);
    }
  }
}
 @Override
 public void updateThrown(Throwable t, String update, Object... args) {
   throw Jvm.rethrow(t);
 }