void _pickInitial() throws MongoException {
    if (_curAddress != null) return;

    // we need to just get a server to query for ismaster
    _pickCurrent();

    try {
      _logger.info("current address beginning of _pickInitial: " + _curAddress);

      DBObject im = isMasterCmd();
      if (_isMaster(im)) return;

      synchronized (_allHosts) {
        Collections.shuffle(_allHosts);
        for (ServerAddress a : _allHosts) {
          if (_curAddress == a) continue;

          _logger.info("remote [" + _curAddress + "] -> [" + a + "]");
          _set(a);

          im = isMasterCmd();
          if (_isMaster(im)) return;

          _logger.severe("switched to: " + a + " but isn't master");
        }

        throw new MongoException("can't find master");
      }
    } catch (Exception e) {
      _logger.log(Level.SEVERE, "can't pick initial master, using random one", e);
    }
  }
Beispiel #2
0
  protected void read(DataInputStream s) {
    try {
      ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
      referencing = Collections.synchronizedSet(new HashSet<String>());
      dirty = new AtomicBoolean(false);
      allocationMode = AllocationMode.valueOf(s.readUTF());
      length = s.readInt();
      Type t = Type.valueOf(s.readUTF());
      if (t == Type.DOUBLE) {
        if (allocationMode == AllocationMode.HEAP) {
          if (this.dataType() == Type.FLOAT) { // DataBuffer type
            // double -> float
            floatData = new float[length()];
          } else if (this.dataType() == Type.DOUBLE) {
            // double -> double
            doubleData = new double[length()];
          } else {
            // double -> int
            intData = new int[length()];
          }
          for (int i = 0; i < length(); i++) {
            put(i, s.readDouble());
          }
        } else {
          wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
          wrappedBuffer.order(ByteOrder.nativeOrder());
          for (int i = 0; i < length(); i++) {
            put(i, s.readDouble());
          }
        }
      } else {
        if (allocationMode == AllocationMode.HEAP) {
          if (this.dataType() == Type.FLOAT) { // DataBuffer type
            // float -> float
            floatData = new float[length()];
          } else if (this.dataType() == Type.DOUBLE) {
            // float -> double
            doubleData = new double[length()];
          } else {
            // float-> int
            intData = new int[length()];
          }
          for (int i = 0; i < length(); i++) {
            put(i, s.readFloat());
          }
        } else {
          wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
          wrappedBuffer.order(ByteOrder.nativeOrder());
          for (int i = 0; i < length(); i++) {
            put(i, s.readFloat());
          }
        }
      }

    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  static void testMixed(Charset cs) throws Throwable {
    CharsetDecoder dec =
        cs.newDecoder()
            .onMalformedInput(CodingErrorAction.REPLACE)
            .onUnmappableCharacter(CodingErrorAction.REPLACE);
    CharsetEncoder enc =
        cs.newEncoder()
            .onMalformedInput(CodingErrorAction.REPLACE)
            .onUnmappableCharacter(CodingErrorAction.REPLACE);
    List<Integer> cps = new ArrayList<>(0x10000);
    int off = 0;
    int cp = 0;
    while (cp < 0x10000) {
      if (enc.canEncode((char) cp)) {
        cps.add(cp);
      }
      cp++;
    }
    Collections.shuffle(cps);
    char[] bmpCA = new char[cps.size()];
    for (int i = 0; i < cps.size(); i++) bmpCA[i] = (char) (int) cps.get(i);
    String bmpStr = new String(bmpCA);
    // getBytes(csn);
    byte[] bmpBA = bmpStr.getBytes(cs.name());
    ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(bmpCA));
    byte[] baNIO = new byte[bf.limit()];
    bf.get(baNIO, 0, baNIO.length);
    if (!Arrays.equals(bmpBA, baNIO)) {
      throw new RuntimeException("getBytes(csn) failed  -> " + cs.name());
    }

    // getBytes(cs);
    bmpBA = bmpStr.getBytes(cs);
    if (!Arrays.equals(bmpBA, baNIO))
      throw new RuntimeException("getBytes(cs) failed  -> " + cs.name());

    // new String(csn);
    String strSC = new String(bmpBA, cs.name());
    String strNIO = dec.reset().decode(ByteBuffer.wrap(bmpBA)).toString();
    if (!strNIO.equals(strSC)) {
      throw new RuntimeException("new String(csn) failed  -> " + cs.name());
    }

    // new String(cs);
    strSC = new String(bmpBA, cs);
    if (!strNIO.equals(strSC)) throw new RuntimeException("new String(cs) failed  -> " + cs.name());
  }
  /**
   * @param fld Folder with files to match.
   * @param ptrn Pattern to match against file name.
   * @return Collection of matched files.
   */
  public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
    List<VisorLogFile> files =
        fileTree(
            fld,
            MAX_FOLDER_DEPTH,
            new FileFilter() {
              @Override
              public boolean accept(File f) {
                return !f.isHidden()
                    && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
              }
            });

    Collections.sort(files, LAST_MODIFIED);

    return files;
  }
Beispiel #5
0
  /**
   * Selects the best text <code>DataFlavor</code> from an array of <code>
   * DataFlavor</code>s. Only <code>DataFlavor.stringFlavor</code>, and equivalent flavors, and
   * flavors that have a primary MIME type of "text", are considered for selection.
   *
   * <p>Flavors are first sorted by their MIME types in the following order:
   *
   * <ul>
   *   <li>"text/sgml"
   *   <li>"text/xml"
   *   <li>"text/html"
   *   <li>"text/rtf"
   *   <li>"text/enriched"
   *   <li>"text/richtext"
   *   <li>"text/uri-list"
   *   <li>"text/tab-separated-values"
   *   <li>"text/t140"
   *   <li>"text/rfc822-headers"
   *   <li>"text/parityfec"
   *   <li>"text/directory"
   *   <li>"text/css"
   *   <li>"text/calendar"
   *   <li>"application/x-java-serialized-object"
   *   <li>"text/plain"
   *   <li>"text/&lt;other&gt;"
   * </ul>
   *
   * <p>For example, "text/sgml" will be selected over "text/html", and <code>
   * DataFlavor.stringFlavor</code> will be chosen over <code>DataFlavor.plainTextFlavor</code>.
   *
   * <p>If two or more flavors share the best MIME type in the array, then that MIME type will be
   * checked to see if it supports the charset parameter.
   *
   * <p>The following MIME types support, or are treated as though they support, the charset
   * parameter:
   *
   * <ul>
   *   <li>"text/sgml"
   *   <li>"text/xml"
   *   <li>"text/html"
   *   <li>"text/enriched"
   *   <li>"text/richtext"
   *   <li>"text/uri-list"
   *   <li>"text/directory"
   *   <li>"text/css"
   *   <li>"text/calendar"
   *   <li>"application/x-java-serialized-object"
   *   <li>"text/plain"
   * </ul>
   *
   * The following MIME types do not support, or are treated as though they do not support, the
   * charset parameter:
   *
   * <ul>
   *   <li>"text/rtf"
   *   <li>"text/tab-separated-values"
   *   <li>"text/t140"
   *   <li>"text/rfc822-headers"
   *   <li>"text/parityfec"
   * </ul>
   *
   * For "text/&lt;other&gt;" MIME types, the first time the JRE needs to determine whether the MIME
   * type supports the charset parameter, it will check whether the parameter is explicitly listed
   * in an arbitrarily chosen <code>DataFlavor</code> which uses that MIME type. If so, the JRE will
   * assume from that point on that the MIME type supports the charset parameter and will not check
   * again. If the parameter is not explicitly listed, the JRE will assume from that point on that
   * the MIME type does not support the charset parameter and will not check again. Because this
   * check is performed on an arbitrarily chosen <code>DataFlavor</code>, developers must ensure
   * that all <code>DataFlavor</code>s with a "text/&lt;other&gt;" MIME type specify the charset
   * parameter if it is supported by that MIME type. Developers should never rely on the JRE to
   * substitute the platform's default charset for a "text/&lt;other&gt;" DataFlavor. Failure to
   * adhere to this restriction will lead to undefined behavior.
   *
   * <p>If the best MIME type in the array does not support the charset parameter, the flavors which
   * share that MIME type will then be sorted by their representation classes in the following
   * order: <code>java.io.InputStream</code>, <code>java.nio.ByteBuffer</code>, <code>[B</code>,
   * &lt;all others&gt;.
   *
   * <p>If two or more flavors share the best representation class, or if no flavor has one of the
   * three specified representations, then one of those flavors will be chosen
   * non-deterministically.
   *
   * <p>If the best MIME type in the array does support the charset parameter, the flavors which
   * share that MIME type will then be sorted by their representation classes in the following
   * order: <code>java.io.Reader</code>, <code>java.lang.String</code>, <code>java.nio.CharBuffer
   * </code>, <code>[C</code>, &lt;all others&gt;.
   *
   * <p>If two or more flavors share the best representation class, and that representation is one
   * of the four explicitly listed, then one of those flavors will be chosen non-deterministically.
   * If, however, no flavor has one of the four specified representations, the flavors will then be
   * sorted by their charsets. Unicode charsets, such as "UTF-16", "UTF-8", "UTF-16BE", "UTF-16LE",
   * and their aliases, are considered best. After them, the platform default charset and its
   * aliases are selected. "US-ASCII" and its aliases are worst. All other charsets are chosen in
   * alphabetical order, but only charsets supported by this implementation of the Java platform
   * will be considered.
   *
   * <p>If two or more flavors share the best charset, the flavors will then again be sorted by
   * their representation classes in the following order: <code>java.io.InputStream</code>, <code>
   * java.nio.ByteBuffer</code>, <code>[B</code>, &lt;all others&gt;.
   *
   * <p>If two or more flavors share the best representation class, or if no flavor has one of the
   * three specified representations, then one of those flavors will be chosen
   * non-deterministically.
   *
   * @param availableFlavors an array of available <code>DataFlavor</code>s
   * @return the best (highest fidelity) flavor according to the rules specified above, or <code>
   *     null</code>, if <code>availableFlavors</code> is <code>null</code>, has zero length, or
   *     contains no text flavors
   * @since 1.3
   */
  public static final DataFlavor selectBestTextFlavor(DataFlavor[] availableFlavors) {
    if (availableFlavors == null || availableFlavors.length == 0) {
      return null;
    }

    if (textFlavorComparator == null) {
      textFlavorComparator = new TextFlavorComparator();
    }

    DataFlavor bestFlavor =
        (DataFlavor) Collections.max(Arrays.asList(availableFlavors), textFlavorComparator);

    if (!bestFlavor.isFlavorTextType()) {
      return null;
    }

    return bestFlavor;
  }
  /**
   * Finds all files in folder and in it's sub-tree of specified depth.
   *
   * @param file Starting folder
   * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
   * @param filter file filter.
   * @return List of found files.
   */
  public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
    if (file.isDirectory()) {
      File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);

      if (files == null) return Collections.emptyList();

      List<VisorLogFile> res = new ArrayList<>(files.length);

      for (File f : files) {
        if (f.isFile() && f.length() > 0) res.add(new VisorLogFile(f));
        else if (maxDepth > 1) res.addAll(fileTree(f, maxDepth - 1, filter));
      }

      return res;
    }

    return F.asList(new VisorLogFile(file));
  }
  private void _pickCurrent() throws MongoException {
    if (_allHosts == null)
      throw new MongoException(
          "got master/slave issue but not in master/slave mode on the client side");

    synchronized (_allHosts) {
      Collections.shuffle(_allHosts);
      for (int i = 0; i < _allHosts.size(); i++) {
        ServerAddress a = _allHosts.get(i);
        if (a == _curAddress) continue;

        if (_curAddress != null) {
          _logger.info("switching from [" + _curAddress + "] to [" + a + "]");
        }

        _set(a);
        return;
      }
    }

    throw new MongoException("couldn't find a new host to swtich too");
  }
 /** @return Near keys. */
 public List<byte[]> nearKeyBytes() {
   return nearKeyBytes != null ? nearKeyBytes : Collections.<byte[]>emptyList();
 }
Beispiel #9
0
public class DBPort {

  public static final int PORT = 27017;
  static final boolean USE_NAGLE = false;

  static final long CONN_RETRY_TIME_MS = 15000;

  public DBPort(InetSocketAddress addr) throws IOException {
    this(addr, null, new MongoOptions());
  }

  DBPort(InetSocketAddress addr, DBPortPool pool, MongoOptions options) throws IOException {
    _options = options;
    _addr = addr;
    _pool = pool;

    _hashCode = _addr.hashCode();

    _logger = Logger.getLogger(_rootLogger.getName() + "." + addr.toString());
  }

  /** @param response will get wiped */
  DBMessage call(DBMessage msg, ByteDecoder decoder) throws IOException {
    return go(msg, decoder);
  }

  void say(DBMessage msg) throws IOException {
    go(msg, null);
  }

  private synchronized DBMessage go(DBMessage msg, ByteDecoder decoder) throws IOException {

    if (_sock == null) _open();

    {
      ByteBuffer out = msg.prepare();
      while (out.remaining() > 0) _sock.write(out);
    }

    if (_pool != null) _pool._everWorked = true;

    if (decoder == null) return null;

    ByteBuffer response = decoder._buf;

    if (response.position() != 0) throw new IllegalArgumentException();

    int read = 0;
    while (read < DBMessage.HEADER_LENGTH) read += _read(response);

    int len = response.getInt(0);
    if (len <= DBMessage.HEADER_LENGTH)
      throw new IllegalArgumentException("db sent invalid length: " + len);

    if (len > response.capacity())
      throw new IllegalArgumentException(
          "db message size is too big (" + len + ") " + "max is (" + response.capacity() + ")");

    response.limit(len);
    while (read < len) read += _read(response);

    if (read != len) throw new RuntimeException("something is wrong");

    response.flip();
    return new DBMessage(response);
  }

  public synchronized void ensureOpen() throws IOException {

    if (_sock != null) return;

    _open();
  }

  void _open() throws IOException {

    long sleepTime = 100;

    final long start = System.currentTimeMillis();
    while (true) {

      IOException lastError = null;

      try {
        _sock = SocketChannel.open();
        _socket = _sock.socket();
        _socket.connect(_addr, _options.connectTimeout);

        _socket.setTcpNoDelay(!USE_NAGLE);
        _socket.setSoTimeout(_options.socketTimeout);
        _in = _socket.getInputStream();
        return;
      } catch (IOException ioe) {
        //  TODO  - erh to fix                lastError = new IOException( "couldn't connect to [" +
        // _addr + "] bc:" + lastError , lastError );
        lastError = new IOException("couldn't connect to [" + _addr + "] bc:" + ioe);
        _logger.log(Level.INFO, "connect fail to : " + _addr, ioe);
      }

      if (!_options.autoConnectRetry || (_pool != null && !_pool._everWorked)) throw lastError;

      long sleptSoFar = System.currentTimeMillis() - start;

      if (sleptSoFar >= CONN_RETRY_TIME_MS) throw lastError;

      if (sleepTime + sleptSoFar > CONN_RETRY_TIME_MS) sleepTime = CONN_RETRY_TIME_MS - sleptSoFar;

      _logger.severe(
          "going to sleep and retry.  total sleep time after = "
              + (sleptSoFar + sleptSoFar)
              + "ms  this time:"
              + sleepTime
              + "ms");
      ThreadUtil.sleep(sleepTime);
      sleepTime *= 2;
    }
  }

  public int hashCode() {
    return _hashCode;
  }

  public String host() {
    return _addr.toString();
  }

  public String toString() {
    return "{DBPort  " + host() + "}";
  }

  protected void finalize() {
    if (_sock != null) {
      try {
        _sock.close();
      } catch (Exception e) {
        // don't care
      }

      _in = null;
      _socket = null;
      _sock = null;
    }
  }

  void checkAuth(DB db) {
    if (db._username == null) return;
    if (_authed.containsKey(db)) return;

    if (_inauth) return;

    _inauth = true;
    try {
      if (db.reauth()) {
        _authed.put(db, true);
        return;
      }
    } finally {
      _inauth = false;
    }

    throw new MongoInternalException("can't reauth!");
  }

  private int _read(ByteBuffer buf) throws IOException {
    int x = _in.read(buf.array(), buf.position(), buf.remaining());
    if (x < 0) throw new IOException("connection to server closed unexpectedly");
    buf.position(buf.position() + x);
    return x;
  }

  final int _hashCode;
  final InetSocketAddress _addr;
  final DBPortPool _pool;
  final MongoOptions _options;
  final Logger _logger;

  private SocketChannel _sock;
  private Socket _socket;
  private InputStream _in;

  private boolean _inauth = false;
  private Map<DB, Boolean> _authed = Collections.synchronizedMap(new WeakHashMap<DB, Boolean>());

  private static Logger _rootLogger = Logger.getLogger("com.mongodb.port");
}
  public static void main(String[] args) throws Throwable {
    final int itrs = Integer.getInteger("iterations", 100000);
    // final int itrs = Integer.getInteger("iterations", 12);
    final int size = Integer.getInteger("size", 2048);
    final int subsize = Integer.getInteger("subsize", 128);
    final int maxchar = Integer.getInteger("maxchar", 128);
    final String regex = System.getProperty("filter");
    final Pattern filter = (regex == null) ? null : Pattern.compile(regex);
    final boolean useSecurityManager = Boolean.getBoolean("SecurityManager");
    if (useSecurityManager) System.setSecurityManager(new PermissiveSecurityManger());
    final Random rnd = new Random();

    String[] csns =
        new String[] {
          "Big5",
          "Johab",
          "EUC_CN",
          "EUC_KR",
          "MS932",
          "MS936",
          "MS949",
          "MS950",
          "GBK",
          "Big5_HKSCS",
          "Big5_HKSCS_2001",
          "Big5_Solaris",
          "MS950_HKSCS",
          "MS950_HKSCS_XP",
          "IBM1364",
          "IBM1381",
          "IBM1383",
          "IBM930",
          "IBM933",
          "IBM935",
          "IBM937",
          "IBM939",
          "IBM942",
          "IBM943",
          "IBM948",
          "IBM949",
          "IBM950",
          "IBM970",
        };

    ArrayList<long[]> sum = new ArrayList<>();

    for (final String csn : csns) {
      final Charset cs = Charset.forName(csn);
      List<Integer> cps = new ArrayList<>(0x4000);
      int off = 0;
      int cp = 0;
      int n = 0;
      CharsetEncoder enc = cs.newEncoder();
      while (cp < 0x10000 && n < cps.size()) {
        if (enc.canEncode((char) cp)) {
          cps.add(cp);
          n++;
        }
        cp++;
      }
      Collections.shuffle(cps);
      char[] ca = new char[cps.size()];
      for (int i = 0; i < cps.size(); i++) ca[i] = (char) (int) cps.get(i);

      System.out.printf("%n--------%s---------%n", csn);
      for (int sz = 8; sz <= 2048; sz *= 2) {
        System.out.printf("   [len=%d]%n", sz);

        final char[] chars = Arrays.copyOf(ca, sz);
        final String str = new String(chars);
        final byte[] bs = str.getBytes(cs);

        Job[] jobs = {
          new Job("String decode: csn") {
            public void work() throws Throwable {
              for (int i = 0; i < itrs; i++) new String(bs, csn);
            }
          },
          new Job("String decode: cs") {
            public void work() throws Throwable {
              for (int i = 0; i < itrs; i++) new String(bs, cs);
            }
          },
          new Job("String encode: csn") {
            public void work() throws Throwable {
              for (int i = 0; i < itrs; i++) str.getBytes(csn);
            }
          },
          new Job("String encode: cs") {
            public void work() throws Throwable {
              for (int i = 0; i < itrs; i++) str.getBytes(cs);
            }
          },
        };
        sum.add(time(jobs));
      }
    }
  }
 /** @return Candidates map. */
 Map<K, Collection<GridCacheDgcLockCandidate>> candidatesMap() {
   return Collections.unmodifiableMap(map);
 }
 /** @return Collection of keys that did not pass the filter. */
 public Collection<IgniteTxKey> filterFailedKeys() {
   return filterFailedKeys == null ? Collections.<IgniteTxKey>emptyList() : filterFailedKeys;
 }
 /** @return Owned values map. */
 public Map<IgniteTxKey, CacheVersionedValue> ownedValues() {
   return ownedVals == null
       ? Collections.<IgniteTxKey, CacheVersionedValue>emptyMap()
       : Collections.unmodifiableMap(ownedVals);
 }
 /**
  * Gets pending versions that are less than {@link #version()}.
  *
  * @return Pending versions.
  */
 public Collection<GridCacheVersion> pending() {
   return pending == null ? Collections.<GridCacheVersion>emptyList() : pending;
 }
Beispiel #15
0
/**
 * Base class for a data buffer handling basic byte operations among other things.
 *
 * @author Adam Gibson
 */
public abstract class BaseDataBuffer implements DataBuffer {

  protected int length;
  protected int offset;
  protected int elementSize;
  protected transient ByteBuffer wrappedBuffer;
  protected Collection<String> referencing = Collections.synchronizedSet(new HashSet<String>());
  protected transient WeakReference<DataBuffer> ref;
  protected boolean isPersist = false;
  protected AllocationMode allocationMode;
  protected double[] doubleData;
  protected int[] intData;
  protected float[] floatData;
  protected AtomicBoolean dirty = new AtomicBoolean(false);
  /**
   * @param buf
   * @param length
   */
  protected BaseDataBuffer(ByteBuf buf, int length, int offset) {
    this(buf, length);
    this.offset = offset;
  }
  /**
   * @param buf
   * @param length
   */
  protected BaseDataBuffer(ByteBuf buf, int length) {
    allocationMode = Nd4j.alloc;
    this.wrappedBuffer = buf.nioBuffer();
    this.length = length;
  }
  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(float[] data, boolean copy, int offset) {
    this(data, copy);
    this.offset = offset;
  }
  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(float[] data, boolean copy) {
    allocationMode = Nd4j.alloc;
    if (allocationMode == AllocationMode.HEAP) {
      if (copy) {
        floatData = ArrayUtil.copy(data);
      } else {
        this.floatData = data;
      }
    } else {
      wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length);
      wrappedBuffer.order(ByteOrder.nativeOrder());
      FloatBuffer buffer = wrappedBuffer.asFloatBuffer();
      for (int i = 0; i < data.length; i++) {
        buffer.put(i, data[i]);
      }
    }
    length = data.length;
  }

  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(double[] data, boolean copy, int offset) {
    this(data, copy);
    this.offset = offset;
  }

  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(double[] data, boolean copy) {
    allocationMode = Nd4j.alloc;
    if (allocationMode == AllocationMode.HEAP) {
      if (copy) {
        doubleData = ArrayUtil.copy(data);
      } else {
        this.doubleData = data;
      }
    } else {
      wrappedBuffer = ByteBuffer.allocateDirect(8 * data.length);
      wrappedBuffer.order(ByteOrder.nativeOrder());
      DoubleBuffer buffer = wrappedBuffer.asDoubleBuffer();
      for (int i = 0; i < data.length; i++) {
        buffer.put(i, data[i]);
      }
    }
    length = data.length;
  }

  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(int[] data, boolean copy, int offset) {
    this(data, copy);
    this.offset = offset;
  }
  /**
   * @param data
   * @param copy
   */
  public BaseDataBuffer(int[] data, boolean copy) {
    allocationMode = Nd4j.alloc;
    if (allocationMode == AllocationMode.HEAP) {
      if (copy) intData = ArrayUtil.copy(data);
      else this.intData = data;

    } else {
      wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length);
      wrappedBuffer.order(ByteOrder.nativeOrder());
      IntBuffer buffer = wrappedBuffer.asIntBuffer();
      for (int i = 0; i < data.length; i++) {
        buffer.put(i, data[i]);
      }
    }
    length = data.length;
  }

  /** @param data */
  public BaseDataBuffer(double[] data) {
    this(data, Nd4j.copyOnOps);
  }

  /** @param data */
  public BaseDataBuffer(int[] data) {
    this(data, Nd4j.copyOnOps);
  }

  /** @param data */
  public BaseDataBuffer(float[] data) {
    this(data, Nd4j.copyOnOps);
  }
  /**
   * @param length
   * @param elementSize
   */
  public BaseDataBuffer(int length, int elementSize, int offset) {
    this(length, elementSize);
    this.offset = offset;
  }

  /**
   * @param length
   * @param elementSize
   */
  public BaseDataBuffer(int length, int elementSize) {
    allocationMode = Nd4j.alloc;
    this.length = length;
    this.elementSize = elementSize;
    if (allocationMode() == AllocationMode.DIRECT) {
      // allows for creation of the nio byte buffer to be overridden
      setNioBuffer();
    } else if (dataType() == Type.DOUBLE) {
      doubleData = new double[length];
    } else if (dataType() == Type.FLOAT) {
      floatData = new float[length];
    } else if (dataType() == Type.INT) intData = new int[length];
  }
  /**
   * Create a data buffer from the given length
   *
   * @param buffer
   * @param length
   */
  public BaseDataBuffer(ByteBuffer buffer, int length, int offset) {
    this(buffer, length);
    this.offset = offset;
  }
  /**
   * Create a data buffer from the given length
   *
   * @param buffer
   * @param length
   */
  public BaseDataBuffer(ByteBuffer buffer, int length) {
    allocationMode = Nd4j.alloc;
    this.length = length;
    buffer.order(ByteOrder.nativeOrder());
    if (allocationMode() == AllocationMode.DIRECT) {
      this.wrappedBuffer = buffer;
    } else if (dataType() == Type.INT) {
      intData = new int[length];
      IntBuffer intBuffer = buffer.asIntBuffer();
      for (int i = 0; i < length; i++) {
        intData[i] = intBuffer.get(i);
      }
    } else if (dataType() == Type.DOUBLE) {
      doubleData = new double[length];
      DoubleBuffer doubleBuffer = buffer.asDoubleBuffer();
      for (int i = 0; i < length; i++) {
        doubleData[i] = doubleBuffer.get(i);
      }

    } else if (dataType() == Type.FLOAT) {
      floatData = new float[length];
      FloatBuffer floatBuffer = buffer.asFloatBuffer();
      for (int i = 0; i < length; i++) {
        floatData[i] = floatBuffer.get(i);
      }
    }
  }

  // sets the nio wrapped buffer (allows to be overridden for other use cases like cuda)
  protected void setNioBuffer() {
    wrappedBuffer = ByteBuffer.allocateDirect(elementSize * length);
    wrappedBuffer.order(ByteOrder.nativeOrder());
  }

  public BaseDataBuffer(byte[] data, int length) {
    this(Unpooled.wrappedBuffer(data), length);
  }

  @Override
  public int offset() {
    return offset;
  }

  @Override
  public AllocationMode allocationMode() {
    return allocationMode;
  }

  @Override
  public void persist() {
    isPersist = true;
  }

  @Override
  public boolean isPersist() {
    return isPersist;
  }

  @Override
  public void unPersist() {
    isPersist = false;
  }

  /**
   * Instantiate a buffer with the given length
   *
   * @param length the length of the buffer
   */
  protected BaseDataBuffer(int length) {
    this.length = length;
    allocationMode = Nd4j.alloc;
    if (length < 0) throw new IllegalArgumentException("Unable to create a buffer of length <= 0");

    ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
    if (allocationMode == AllocationMode.HEAP) {
      if (length >= Integer.MAX_VALUE)
        throw new IllegalArgumentException(
            "Length of data buffer can not be > Integer.MAX_VALUE for heap (array based storage) allocation");
      if (dataType() == Type.DOUBLE) doubleData = new double[length];
      else if (dataType() == Type.FLOAT) floatData = new float[length];
    } else {
      if (length * getElementSize() < 0)
        throw new IllegalArgumentException(
            "Unable to create buffer of length " + length + " due to negative length specified");
      wrappedBuffer =
          ByteBuffer.allocateDirect(getElementSize() * length).order(ByteOrder.nativeOrder());
    }
  }

  @Override
  public void copyAtStride(
      DataBuffer buf, int n, int stride, int yStride, int offset, int yOffset) {
    if (dataType() == Type.FLOAT) {
      for (int i = 0; i < n; i++) {
        put(offset + i * stride, buf.getFloat(yOffset + i * yStride));
      }
    } else {
      for (int i = 0; i < n; i++) {
        put(offset + i * stride, buf.getDouble(yOffset + i * yStride));
      }
    }
  }

  @Override
  public void removeReferencing(String id) {
    referencing.remove(id);
  }

  @Override
  public Collection<String> references() {
    return referencing;
  }

  @Override
  public void addReferencing(String id) {
    referencing.add(id);
  }

  @Override
  public void assign(int[] indices, float[] data, boolean contiguous, int inc) {
    if (indices.length != data.length)
      throw new IllegalArgumentException("Indices and data length must be the same");
    if (indices.length > length())
      throw new IllegalArgumentException(
          "More elements than space to assign. This buffer is of length "
              + length()
              + " where the indices are of length "
              + data.length);
    for (int i = 0; i < indices.length; i++) {
      put(indices[i], data[i]);
    }
  }

  @Override
  public void setData(int[] data) {
    if (intData != null) this.intData = data;
    else {
      for (int i = 0; i < data.length; i++) {
        put(i, data[i]);
      }
    }
  }

  @Override
  public void setData(float[] data) {
    if (floatData != null) {
      this.floatData = data;
    } else {
      for (int i = 0; i < data.length; i++) put(i, data[i]);
    }
  }

  @Override
  public void setData(double[] data) {
    if (doubleData != null) {
      this.doubleData = data;
    } else {
      for (int i = 0; i < data.length; i++) put(i, data[i]);
    }
  }

  @Override
  public void assign(int[] indices, double[] data, boolean contiguous, int inc) {
    if (indices.length != data.length)
      throw new IllegalArgumentException("Indices and data length must be the same");
    if (indices.length > length())
      throw new IllegalArgumentException(
          "More elements than space to assign. This buffer is of length "
              + length()
              + " where the indices are of length "
              + data.length);
    for (int i = 0; i < indices.length; i += inc) {
      put(indices[i], data[i]);
    }
  }

  @Override
  public void assign(DataBuffer data) {
    if (data.length() != length())
      throw new IllegalArgumentException(
          "Unable to assign buffer of length "
              + data.length()
              + " to this buffer of length "
              + length());

    for (int i = 0; i < data.length(); i++) {
      put(i, data.getDouble(i));
    }
  }

  @Override
  public void assign(int[] indices, float[] data, boolean contiguous) {
    assign(indices, data, contiguous, 1);
  }

  @Override
  public void assign(int[] indices, double[] data, boolean contiguous) {
    assign(indices, data, contiguous, 1);
  }

  @Override
  public int length() {
    return length;
  }

  @Override
  public void assign(Number value) {
    for (int i = 0; i < length(); i++) assign(value, i);
  }

  @Override
  public double[] getDoublesAt(int offset, int length) {
    return getDoublesAt(offset, 1, length);
  }

  @Override
  public float[] getFloatsAt(int offset, int inc, int length) {
    if (offset + length > length()) length -= offset;
    float[] ret = new float[length];
    for (int i = 0; i < length; i++) {
      ret[i] = getFloat(i + offset);
    }
    return ret;
  }

  @Override
  public DataBuffer dup() {
    if (floatData != null) {
      return create(floatData);
    } else if (doubleData != null) {
      return create(doubleData);
    } else if (intData != null) {
      return create(intData);
    }

    DataBuffer ret = create(length);
    for (int i = 0; i < ret.length(); i++) ret.put(i, getDouble(i));

    return ret;
  }

  /**
   * Create with length
   *
   * @param length a databuffer of the same type as this with the given length
   * @return a data buffer with the same length and datatype as this one
   */
  protected abstract DataBuffer create(int length);

  /**
   * Create the data buffer with respect to the given byte buffer
   *
   * @param data the buffer to create
   * @return the data buffer based on the given buffer
   */
  public abstract DataBuffer create(double[] data);
  /**
   * Create the data buffer with respect to the given byte buffer
   *
   * @param data the buffer to create
   * @return the data buffer based on the given buffer
   */
  public abstract DataBuffer create(float[] data);

  /**
   * Create the data buffer with respect to the given byte buffer
   *
   * @param data the buffer to create
   * @return the data buffer based on the given buffer
   */
  public abstract DataBuffer create(int[] data);

  /**
   * Create the data buffer with respect to the given byte buffer
   *
   * @param buf the buffer to create
   * @return the data buffer based on the given buffer
   */
  public abstract DataBuffer create(ByteBuf buf, int length);

  @Override
  public double[] getDoublesAt(int offset, int inc, int length) {
    if (offset + length > length()) length -= offset;

    double[] ret = new double[length];
    for (int i = 0; i < length; i++) {
      ret[i] = getDouble(i + offset);
    }

    return ret;
  }

  @Override
  public float[] getFloatsAt(int offset, int length) {
    return getFloatsAt(offset, 1, length);
  }

  @Override
  public IComplexFloat getComplexFloat(int i) {
    return Nd4j.createFloat(getFloat(i), getFloat(i + 1));
  }

  @Override
  public IComplexDouble getComplexDouble(int i) {
    return Nd4j.createDouble(getDouble(i), getDouble(i + 1));
  }

  @Override
  public IComplexNumber getComplex(int i) {
    return dataType() == DataBuffer.Type.FLOAT ? getComplexFloat(i) : getComplexDouble(i);
  }

  @Override
  public void put(int i, IComplexNumber result) {
    put(i, result.realComponent().doubleValue());
    put(i + 1, result.imaginaryComponent().doubleValue());
  }

  @Override
  public void assign(int[] offsets, int[] strides, DataBuffer... buffers) {
    assign(offsets, strides, length(), buffers);
  }

  @Override
  public byte[] asBytes() {
    if (allocationMode == AllocationMode.HEAP) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(getElementSize() * length());
      DataOutputStream dos = new DataOutputStream(bos);

      if (dataType() == Type.DOUBLE) {
        if (doubleData == null) throw new IllegalStateException("Double array is null!");

        try {
          for (int i = 0; i < doubleData.length; i++) dos.writeDouble(doubleData[i]);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }

      } else {
        if (floatData == null) throw new IllegalStateException("Double array is null!");

        try {
          for (int i = 0; i < floatData.length; i++) dos.writeFloat(floatData[i]);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }

      return bos.toByteArray();

    } else {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      DataOutputStream dos = new DataOutputStream(bos);
      if (dataType() == Type.DOUBLE) {
        for (int i = 0; i < length(); i++) {
          try {
            dos.writeDouble(getDouble(i));
          } catch (IOException e) {
            e.printStackTrace();
          }
        }
      } else {
        for (int i = 0; i < length(); i++) {
          try {
            dos.writeFloat(getFloat(i));
          } catch (IOException e) {
            e.printStackTrace();
          }
        }
      }
      return bos.toByteArray();
    }
  }

  @Override
  public float[] asFloat() {
    if (allocationMode == AllocationMode.HEAP) {
      if (floatData != null) {
        return floatData;
      }
    }

    float[] ret = new float[length];
    for (int i = 0; i < length; i++) ret[i] = getFloat(i);
    return ret;
  }

  @Override
  public double[] asDouble() {
    if (allocationMode == AllocationMode.HEAP) {
      if (doubleData != null) {
        return doubleData;
      }
    }

    double[] ret = new double[length];
    for (int i = 0; i < length; i++) ret[i] = getDouble(i);
    return ret;
  }

  @Override
  public int[] asInt() {
    if (allocationMode == AllocationMode.HEAP) {
      if (intData != null) {
        return intData;
      }
    }
    return wrappedBuffer.asIntBuffer().array();
  }

  @Override
  public double getDouble(int i) {
    if (doubleData != null) {
      if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i);
      dirty.set(false);
      return doubleData[i];
    } else if (floatData != null) {
      if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i);
      dirty.set(false);
      return (double) floatData[i];
    } else if (intData != null) {
      dirty.set(false);
      return (double) intData[i];
    }

    if (dataType() == Type.FLOAT) {
      dirty.set(false);
      return wrappedBuffer.asFloatBuffer().get(i);
    } else if (dataType() == Type.INT) {
      dirty.set(false);
      return wrappedBuffer.asIntBuffer().get(i);
    } else {
      dirty.set(false);
      return wrappedBuffer.asDoubleBuffer().get(i);
    }
  }

  @Override
  public float getFloat(int i) {
    if (doubleData != null) {
      if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i);
      dirty.set(false);
      return (float) doubleData[i];
    } else if (floatData != null) {
      if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i);
      dirty.set(false);
      return floatData[i];
    } else if (intData != null) {
      dirty.set(false);
      return (float) intData[i];
    }

    if (dataType() == Type.DOUBLE) {
      dirty.set(false);
      return (float) wrappedBuffer.asDoubleBuffer().get(i);
    }

    dirty.getAndSet(true);
    return wrappedBuffer.asFloatBuffer().get(i);
  }

  @Override
  public Number getNumber(int i) {
    if (dataType() == Type.DOUBLE) return getDouble(i);
    else if (dataType() == Type.INT) return getInt(i);
    return getFloat(i);
  }

  @Override
  public void put(int i, float element) {
    put(i, (double) element);
  }

  @Override
  public void put(int i, double element) {
    if (i < 0 || i >= length()) throw new IllegalArgumentException("Illegal index " + i);

    if (doubleData != null) doubleData[i] = element;
    else if (floatData != null) floatData[i] = (float) element;
    else if (intData != null) intData[i] = (int) element;
    else {
      if (dataType() == Type.DOUBLE) {
        wrappedBuffer.asDoubleBuffer().put(i, element);

      } else if (dataType() == Type.INT) {
        wrappedBuffer.asIntBuffer().put(i, (int) element);
      } else {
        wrappedBuffer.asFloatBuffer().put(i, (float) element);
      }
    }

    dirty.set(true);
  }

  @Override
  public boolean dirty() {
    return dirty.get();
  }

  @Override
  public IntBuffer asNioInt() {
    if (wrappedBuffer == null) {
      return IntBuffer.wrap(intData);
    }
    return wrappedBuffer.asIntBuffer();
  }

  @Override
  public DoubleBuffer asNioDouble() {
    if (wrappedBuffer == null) {
      return DoubleBuffer.wrap(doubleData);
    }
    return wrappedBuffer.asDoubleBuffer();
  }

  @Override
  public FloatBuffer asNioFloat() {
    if (wrappedBuffer == null) {
      return FloatBuffer.wrap(floatData);
    }
    return wrappedBuffer.asFloatBuffer();
  }

  @Override
  public ByteBuffer asNio() {
    return wrappedBuffer;
  }

  @Override
  public ByteBuf asNetty() {
    if (wrappedBuffer != null) return Unpooled.wrappedBuffer(wrappedBuffer);
    else if (floatData != null) return Unpooled.copyFloat(floatData);
    else if (doubleData != null) return Unpooled.copyDouble(doubleData);
    throw new IllegalStateException("No data source defined");
  }

  @Override
  public void put(int i, int element) {
    put(i, (double) element);
  }

  @Override
  public void assign(Number value, int offset) {
    for (int i = offset; i < length(); i++) put(i, value.doubleValue());
  }

  @Override
  public void write(OutputStream dos) {
    if (dos instanceof DataOutputStream) {
      try {
        write((DataOutputStream) dos);
      } catch (IOException e) {
        e.printStackTrace();
      }
    } else {
      DataOutputStream dos2 = new DataOutputStream(dos);
      try {

        write(dos2);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }

  @Override
  public void read(InputStream is) {
    if (is instanceof DataInputStream) {
      read((DataInputStream) is);
    } else {
      DataInputStream dis2 = new DataInputStream(is);
      read(dis2);
    }
  }

  @Override
  public void flush() {}

  @Override
  public int getInt(int ix) {
    return (int) getDouble(ix);
  }

  @Override
  public void assign(int[] offsets, int[] strides, long n, DataBuffer... buffers) {
    if (offsets.length != strides.length || strides.length != buffers.length)
      throw new IllegalArgumentException(
          "Unable to assign buffers, please specify equal lengths strides, offsets, and buffers");
    int length = 0;
    for (int i = 0; i < buffers.length; i++) length += buffers[i].length();

    int count = 0;
    for (int i = 0; i < buffers.length; i++) {
      for (int j = offsets[i]; j < buffers[i].length(); j += strides[i]) {
        put(count++, buffers[i].getDouble(j));
      }
    }

    if (count != n)
      throw new IllegalArgumentException("Strides and offsets didn't match up to length " + n);
  }

  @Override
  public void assign(DataBuffer... buffers) {
    int[] offsets = new int[buffers.length];
    int[] strides = new int[buffers.length];
    for (int i = 0; i < strides.length; i++) strides[i] = 1;
    assign(offsets, strides, buffers);
  }

  @Override
  public void destroy() {}

  @Override
  public boolean equals(Object o) {
    if (o instanceof DataBuffer) {
      DataBuffer d = (DataBuffer) o;
      if (d.length() != length()) return false;
      for (int i = 0; i < length(); i++) {
        double eps = Math.abs(getDouble(i) - d.getDouble(i));
        if (eps > Nd4j.EPS_THRESHOLD) return false;
      }
    }

    return true;
  }

  private void readObject(ObjectInputStream s) {
    doReadObject(s);
  }

  private void writeObject(java.io.ObjectOutputStream out) throws IOException {
    out.defaultWriteObject();
    write(out);
  }

  protected void doReadObject(ObjectInputStream s) {
    try {
      s.defaultReadObject();
      read(s);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  protected void read(DataInputStream s) {
    try {
      ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
      referencing = Collections.synchronizedSet(new HashSet<String>());
      dirty = new AtomicBoolean(false);
      allocationMode = AllocationMode.valueOf(s.readUTF());
      length = s.readInt();
      Type t = Type.valueOf(s.readUTF());
      if (t == Type.DOUBLE) {
        if (allocationMode == AllocationMode.HEAP) {
          if (this.dataType() == Type.FLOAT) { // DataBuffer type
            // double -> float
            floatData = new float[length()];
          } else if (this.dataType() == Type.DOUBLE) {
            // double -> double
            doubleData = new double[length()];
          } else {
            // double -> int
            intData = new int[length()];
          }
          for (int i = 0; i < length(); i++) {
            put(i, s.readDouble());
          }
        } else {
          wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
          wrappedBuffer.order(ByteOrder.nativeOrder());
          for (int i = 0; i < length(); i++) {
            put(i, s.readDouble());
          }
        }
      } else {
        if (allocationMode == AllocationMode.HEAP) {
          if (this.dataType() == Type.FLOAT) { // DataBuffer type
            // float -> float
            floatData = new float[length()];
          } else if (this.dataType() == Type.DOUBLE) {
            // float -> double
            doubleData = new double[length()];
          } else {
            // float-> int
            intData = new int[length()];
          }
          for (int i = 0; i < length(); i++) {
            put(i, s.readFloat());
          }
        } else {
          wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
          wrappedBuffer.order(ByteOrder.nativeOrder());
          for (int i = 0; i < length(); i++) {
            put(i, s.readFloat());
          }
        }
      }

    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  protected void write(DataOutputStream out) throws IOException {
    out.writeUTF(allocationMode.name());
    out.writeInt(length());
    out.writeUTF(dataType().name());
    if (dataType() == Type.DOUBLE) {
      for (int i = 0; i < length(); i++) out.writeDouble(getDouble(i));
    } else {
      for (int i = 0; i < length(); i++) out.writeFloat(getFloat(i));
    }
  }

  @Override
  public Object array() {
    if (floatData != null) return floatData;
    if (doubleData != null) return doubleData;
    throw new UnsupportedOperationException();
  }

  @Override
  public String toString() {
    StringBuffer ret = new StringBuffer();
    ret.append("[");
    for (int i = 0; i < length(); i++) {
      ret.append(getNumber(i));
      if (i < length() - 1) ret.append(",");
    }
    ret.append("]");

    return ret.toString();
  }

  @Override
  public int hashCode() {
    int result = length;
    result = 31 * result + (referencing != null ? referencing.hashCode() : 0);
    result = 31 * result + (ref != null ? ref.hashCode() : 0);
    result = 31 * result + (isPersist ? 1 : 0);
    result = 31 * result + (allocationMode != null ? allocationMode.hashCode() : 0);
    return result;
  }
}
 /** @return Map of versions to be verified. */
 public Map<IgniteTxKey, GridCacheVersion> dhtVersions() {
   return dhtVers == null ? Collections.<IgniteTxKey, GridCacheVersion>emptyMap() : dhtVers;
 }
  /**
   * Grabs local events and detects if events was lost since last poll.
   *
   * @param ignite Target grid.
   * @param evtOrderKey Unique key to take last order key from node local map.
   * @param evtThrottleCntrKey Unique key to take throttle count from node local map.
   * @param evtTypes Event types to collect.
   * @param evtMapper Closure to map grid events to Visor data transfer objects.
   * @return Collections of node events
   */
  public static Collection<VisorGridEvent> collectEvents(
      Ignite ignite,
      String evtOrderKey,
      String evtThrottleCntrKey,
      final int[] evtTypes,
      IgniteClosure<Event, VisorGridEvent> evtMapper) {
    assert ignite != null;
    assert evtTypes != null && evtTypes.length > 0;

    ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();

    final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
    final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);

    // When we first time arrive onto a node to get its local events,
    // we'll grab only last those events that not older than given period to make sure we are
    // not grabbing GBs of data accidentally.
    final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;

    // Flag for detecting gaps between events.
    final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);

    IgnitePredicate<Event> p =
        new IgnitePredicate<Event>() {
          /** */
          private static final long serialVersionUID = 0L;

          @Override
          public boolean apply(Event e) {
            // Detects that events were lost.
            if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);

            // Retains events by lastOrder, period and type.
            return e.localOrder() > lastOrder
                && e.timestamp() > notOlderThan
                && F.contains(evtTypes, e.type());
          }
        };

    Collection<Event> evts = ignite.events().localQuery(p);

    // Update latest order in node local, if not empty.
    if (!evts.isEmpty()) {
      Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);

      nl.put(evtOrderKey, maxEvt.localOrder());
    }

    // Update throttle counter.
    if (!lastFound.get())
      nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);

    boolean lost = !lastFound.get() && throttle == 0;

    Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));

    if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));

    for (Event e : evts) {
      VisorGridEvent visorEvt = evtMapper.apply(e);

      if (visorEvt != null) res.add(visorEvt);
    }

    return res;
  }