void _pickInitial() throws MongoException { if (_curAddress != null) return; // we need to just get a server to query for ismaster _pickCurrent(); try { _logger.info("current address beginning of _pickInitial: " + _curAddress); DBObject im = isMasterCmd(); if (_isMaster(im)) return; synchronized (_allHosts) { Collections.shuffle(_allHosts); for (ServerAddress a : _allHosts) { if (_curAddress == a) continue; _logger.info("remote [" + _curAddress + "] -> [" + a + "]"); _set(a); im = isMasterCmd(); if (_isMaster(im)) return; _logger.severe("switched to: " + a + " but isn't master"); } throw new MongoException("can't find master"); } } catch (Exception e) { _logger.log(Level.SEVERE, "can't pick initial master, using random one", e); } }
protected void read(DataInputStream s) { try { ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue()); referencing = Collections.synchronizedSet(new HashSet<String>()); dirty = new AtomicBoolean(false); allocationMode = AllocationMode.valueOf(s.readUTF()); length = s.readInt(); Type t = Type.valueOf(s.readUTF()); if (t == Type.DOUBLE) { if (allocationMode == AllocationMode.HEAP) { if (this.dataType() == Type.FLOAT) { // DataBuffer type // double -> float floatData = new float[length()]; } else if (this.dataType() == Type.DOUBLE) { // double -> double doubleData = new double[length()]; } else { // double -> int intData = new int[length()]; } for (int i = 0; i < length(); i++) { put(i, s.readDouble()); } } else { wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize()); wrappedBuffer.order(ByteOrder.nativeOrder()); for (int i = 0; i < length(); i++) { put(i, s.readDouble()); } } } else { if (allocationMode == AllocationMode.HEAP) { if (this.dataType() == Type.FLOAT) { // DataBuffer type // float -> float floatData = new float[length()]; } else if (this.dataType() == Type.DOUBLE) { // float -> double doubleData = new double[length()]; } else { // float-> int intData = new int[length()]; } for (int i = 0; i < length(); i++) { put(i, s.readFloat()); } } else { wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize()); wrappedBuffer.order(ByteOrder.nativeOrder()); for (int i = 0; i < length(); i++) { put(i, s.readFloat()); } } } } catch (Exception e) { throw new RuntimeException(e); } }
static void testMixed(Charset cs) throws Throwable { CharsetDecoder dec = cs.newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); CharsetEncoder enc = cs.newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); List<Integer> cps = new ArrayList<>(0x10000); int off = 0; int cp = 0; while (cp < 0x10000) { if (enc.canEncode((char) cp)) { cps.add(cp); } cp++; } Collections.shuffle(cps); char[] bmpCA = new char[cps.size()]; for (int i = 0; i < cps.size(); i++) bmpCA[i] = (char) (int) cps.get(i); String bmpStr = new String(bmpCA); // getBytes(csn); byte[] bmpBA = bmpStr.getBytes(cs.name()); ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(bmpCA)); byte[] baNIO = new byte[bf.limit()]; bf.get(baNIO, 0, baNIO.length); if (!Arrays.equals(bmpBA, baNIO)) { throw new RuntimeException("getBytes(csn) failed -> " + cs.name()); } // getBytes(cs); bmpBA = bmpStr.getBytes(cs); if (!Arrays.equals(bmpBA, baNIO)) throw new RuntimeException("getBytes(cs) failed -> " + cs.name()); // new String(csn); String strSC = new String(bmpBA, cs.name()); String strNIO = dec.reset().decode(ByteBuffer.wrap(bmpBA)).toString(); if (!strNIO.equals(strSC)) { throw new RuntimeException("new String(csn) failed -> " + cs.name()); } // new String(cs); strSC = new String(bmpBA, cs); if (!strNIO.equals(strSC)) throw new RuntimeException("new String(cs) failed -> " + cs.name()); }
/** * @param fld Folder with files to match. * @param ptrn Pattern to match against file name. * @return Collection of matched files. */ public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) { List<VisorLogFile> files = fileTree( fld, MAX_FOLDER_DEPTH, new FileFilter() { @Override public boolean accept(File f) { return !f.isHidden() && (f.isDirectory() || f.isFile() && f.getName().matches(ptrn)); } }); Collections.sort(files, LAST_MODIFIED); return files; }
/** * Selects the best text <code>DataFlavor</code> from an array of <code> * DataFlavor</code>s. Only <code>DataFlavor.stringFlavor</code>, and equivalent flavors, and * flavors that have a primary MIME type of "text", are considered for selection. * * <p>Flavors are first sorted by their MIME types in the following order: * * <ul> * <li>"text/sgml" * <li>"text/xml" * <li>"text/html" * <li>"text/rtf" * <li>"text/enriched" * <li>"text/richtext" * <li>"text/uri-list" * <li>"text/tab-separated-values" * <li>"text/t140" * <li>"text/rfc822-headers" * <li>"text/parityfec" * <li>"text/directory" * <li>"text/css" * <li>"text/calendar" * <li>"application/x-java-serialized-object" * <li>"text/plain" * <li>"text/<other>" * </ul> * * <p>For example, "text/sgml" will be selected over "text/html", and <code> * DataFlavor.stringFlavor</code> will be chosen over <code>DataFlavor.plainTextFlavor</code>. * * <p>If two or more flavors share the best MIME type in the array, then that MIME type will be * checked to see if it supports the charset parameter. * * <p>The following MIME types support, or are treated as though they support, the charset * parameter: * * <ul> * <li>"text/sgml" * <li>"text/xml" * <li>"text/html" * <li>"text/enriched" * <li>"text/richtext" * <li>"text/uri-list" * <li>"text/directory" * <li>"text/css" * <li>"text/calendar" * <li>"application/x-java-serialized-object" * <li>"text/plain" * </ul> * * The following MIME types do not support, or are treated as though they do not support, the * charset parameter: * * <ul> * <li>"text/rtf" * <li>"text/tab-separated-values" * <li>"text/t140" * <li>"text/rfc822-headers" * <li>"text/parityfec" * </ul> * * For "text/<other>" MIME types, the first time the JRE needs to determine whether the MIME * type supports the charset parameter, it will check whether the parameter is explicitly listed * in an arbitrarily chosen <code>DataFlavor</code> which uses that MIME type. If so, the JRE will * assume from that point on that the MIME type supports the charset parameter and will not check * again. If the parameter is not explicitly listed, the JRE will assume from that point on that * the MIME type does not support the charset parameter and will not check again. Because this * check is performed on an arbitrarily chosen <code>DataFlavor</code>, developers must ensure * that all <code>DataFlavor</code>s with a "text/<other>" MIME type specify the charset * parameter if it is supported by that MIME type. Developers should never rely on the JRE to * substitute the platform's default charset for a "text/<other>" DataFlavor. Failure to * adhere to this restriction will lead to undefined behavior. * * <p>If the best MIME type in the array does not support the charset parameter, the flavors which * share that MIME type will then be sorted by their representation classes in the following * order: <code>java.io.InputStream</code>, <code>java.nio.ByteBuffer</code>, <code>[B</code>, * <all others>. * * <p>If two or more flavors share the best representation class, or if no flavor has one of the * three specified representations, then one of those flavors will be chosen * non-deterministically. * * <p>If the best MIME type in the array does support the charset parameter, the flavors which * share that MIME type will then be sorted by their representation classes in the following * order: <code>java.io.Reader</code>, <code>java.lang.String</code>, <code>java.nio.CharBuffer * </code>, <code>[C</code>, <all others>. * * <p>If two or more flavors share the best representation class, and that representation is one * of the four explicitly listed, then one of those flavors will be chosen non-deterministically. * If, however, no flavor has one of the four specified representations, the flavors will then be * sorted by their charsets. Unicode charsets, such as "UTF-16", "UTF-8", "UTF-16BE", "UTF-16LE", * and their aliases, are considered best. After them, the platform default charset and its * aliases are selected. "US-ASCII" and its aliases are worst. All other charsets are chosen in * alphabetical order, but only charsets supported by this implementation of the Java platform * will be considered. * * <p>If two or more flavors share the best charset, the flavors will then again be sorted by * their representation classes in the following order: <code>java.io.InputStream</code>, <code> * java.nio.ByteBuffer</code>, <code>[B</code>, <all others>. * * <p>If two or more flavors share the best representation class, or if no flavor has one of the * three specified representations, then one of those flavors will be chosen * non-deterministically. * * @param availableFlavors an array of available <code>DataFlavor</code>s * @return the best (highest fidelity) flavor according to the rules specified above, or <code> * null</code>, if <code>availableFlavors</code> is <code>null</code>, has zero length, or * contains no text flavors * @since 1.3 */ public static final DataFlavor selectBestTextFlavor(DataFlavor[] availableFlavors) { if (availableFlavors == null || availableFlavors.length == 0) { return null; } if (textFlavorComparator == null) { textFlavorComparator = new TextFlavorComparator(); } DataFlavor bestFlavor = (DataFlavor) Collections.max(Arrays.asList(availableFlavors), textFlavorComparator); if (!bestFlavor.isFlavorTextType()) { return null; } return bestFlavor; }
/** * Finds all files in folder and in it's sub-tree of specified depth. * * @param file Starting folder * @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders. * @param filter file filter. * @return List of found files. */ public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) { if (file.isDirectory()) { File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter); if (files == null) return Collections.emptyList(); List<VisorLogFile> res = new ArrayList<>(files.length); for (File f : files) { if (f.isFile() && f.length() > 0) res.add(new VisorLogFile(f)); else if (maxDepth > 1) res.addAll(fileTree(f, maxDepth - 1, filter)); } return res; } return F.asList(new VisorLogFile(file)); }
private void _pickCurrent() throws MongoException { if (_allHosts == null) throw new MongoException( "got master/slave issue but not in master/slave mode on the client side"); synchronized (_allHosts) { Collections.shuffle(_allHosts); for (int i = 0; i < _allHosts.size(); i++) { ServerAddress a = _allHosts.get(i); if (a == _curAddress) continue; if (_curAddress != null) { _logger.info("switching from [" + _curAddress + "] to [" + a + "]"); } _set(a); return; } } throw new MongoException("couldn't find a new host to swtich too"); }
/** @return Near keys. */ public List<byte[]> nearKeyBytes() { return nearKeyBytes != null ? nearKeyBytes : Collections.<byte[]>emptyList(); }
public class DBPort { public static final int PORT = 27017; static final boolean USE_NAGLE = false; static final long CONN_RETRY_TIME_MS = 15000; public DBPort(InetSocketAddress addr) throws IOException { this(addr, null, new MongoOptions()); } DBPort(InetSocketAddress addr, DBPortPool pool, MongoOptions options) throws IOException { _options = options; _addr = addr; _pool = pool; _hashCode = _addr.hashCode(); _logger = Logger.getLogger(_rootLogger.getName() + "." + addr.toString()); } /** @param response will get wiped */ DBMessage call(DBMessage msg, ByteDecoder decoder) throws IOException { return go(msg, decoder); } void say(DBMessage msg) throws IOException { go(msg, null); } private synchronized DBMessage go(DBMessage msg, ByteDecoder decoder) throws IOException { if (_sock == null) _open(); { ByteBuffer out = msg.prepare(); while (out.remaining() > 0) _sock.write(out); } if (_pool != null) _pool._everWorked = true; if (decoder == null) return null; ByteBuffer response = decoder._buf; if (response.position() != 0) throw new IllegalArgumentException(); int read = 0; while (read < DBMessage.HEADER_LENGTH) read += _read(response); int len = response.getInt(0); if (len <= DBMessage.HEADER_LENGTH) throw new IllegalArgumentException("db sent invalid length: " + len); if (len > response.capacity()) throw new IllegalArgumentException( "db message size is too big (" + len + ") " + "max is (" + response.capacity() + ")"); response.limit(len); while (read < len) read += _read(response); if (read != len) throw new RuntimeException("something is wrong"); response.flip(); return new DBMessage(response); } public synchronized void ensureOpen() throws IOException { if (_sock != null) return; _open(); } void _open() throws IOException { long sleepTime = 100; final long start = System.currentTimeMillis(); while (true) { IOException lastError = null; try { _sock = SocketChannel.open(); _socket = _sock.socket(); _socket.connect(_addr, _options.connectTimeout); _socket.setTcpNoDelay(!USE_NAGLE); _socket.setSoTimeout(_options.socketTimeout); _in = _socket.getInputStream(); return; } catch (IOException ioe) { // TODO - erh to fix lastError = new IOException( "couldn't connect to [" + // _addr + "] bc:" + lastError , lastError ); lastError = new IOException("couldn't connect to [" + _addr + "] bc:" + ioe); _logger.log(Level.INFO, "connect fail to : " + _addr, ioe); } if (!_options.autoConnectRetry || (_pool != null && !_pool._everWorked)) throw lastError; long sleptSoFar = System.currentTimeMillis() - start; if (sleptSoFar >= CONN_RETRY_TIME_MS) throw lastError; if (sleepTime + sleptSoFar > CONN_RETRY_TIME_MS) sleepTime = CONN_RETRY_TIME_MS - sleptSoFar; _logger.severe( "going to sleep and retry. total sleep time after = " + (sleptSoFar + sleptSoFar) + "ms this time:" + sleepTime + "ms"); ThreadUtil.sleep(sleepTime); sleepTime *= 2; } } public int hashCode() { return _hashCode; } public String host() { return _addr.toString(); } public String toString() { return "{DBPort " + host() + "}"; } protected void finalize() { if (_sock != null) { try { _sock.close(); } catch (Exception e) { // don't care } _in = null; _socket = null; _sock = null; } } void checkAuth(DB db) { if (db._username == null) return; if (_authed.containsKey(db)) return; if (_inauth) return; _inauth = true; try { if (db.reauth()) { _authed.put(db, true); return; } } finally { _inauth = false; } throw new MongoInternalException("can't reauth!"); } private int _read(ByteBuffer buf) throws IOException { int x = _in.read(buf.array(), buf.position(), buf.remaining()); if (x < 0) throw new IOException("connection to server closed unexpectedly"); buf.position(buf.position() + x); return x; } final int _hashCode; final InetSocketAddress _addr; final DBPortPool _pool; final MongoOptions _options; final Logger _logger; private SocketChannel _sock; private Socket _socket; private InputStream _in; private boolean _inauth = false; private Map<DB, Boolean> _authed = Collections.synchronizedMap(new WeakHashMap<DB, Boolean>()); private static Logger _rootLogger = Logger.getLogger("com.mongodb.port"); }
public static void main(String[] args) throws Throwable { final int itrs = Integer.getInteger("iterations", 100000); // final int itrs = Integer.getInteger("iterations", 12); final int size = Integer.getInteger("size", 2048); final int subsize = Integer.getInteger("subsize", 128); final int maxchar = Integer.getInteger("maxchar", 128); final String regex = System.getProperty("filter"); final Pattern filter = (regex == null) ? null : Pattern.compile(regex); final boolean useSecurityManager = Boolean.getBoolean("SecurityManager"); if (useSecurityManager) System.setSecurityManager(new PermissiveSecurityManger()); final Random rnd = new Random(); String[] csns = new String[] { "Big5", "Johab", "EUC_CN", "EUC_KR", "MS932", "MS936", "MS949", "MS950", "GBK", "Big5_HKSCS", "Big5_HKSCS_2001", "Big5_Solaris", "MS950_HKSCS", "MS950_HKSCS_XP", "IBM1364", "IBM1381", "IBM1383", "IBM930", "IBM933", "IBM935", "IBM937", "IBM939", "IBM942", "IBM943", "IBM948", "IBM949", "IBM950", "IBM970", }; ArrayList<long[]> sum = new ArrayList<>(); for (final String csn : csns) { final Charset cs = Charset.forName(csn); List<Integer> cps = new ArrayList<>(0x4000); int off = 0; int cp = 0; int n = 0; CharsetEncoder enc = cs.newEncoder(); while (cp < 0x10000 && n < cps.size()) { if (enc.canEncode((char) cp)) { cps.add(cp); n++; } cp++; } Collections.shuffle(cps); char[] ca = new char[cps.size()]; for (int i = 0; i < cps.size(); i++) ca[i] = (char) (int) cps.get(i); System.out.printf("%n--------%s---------%n", csn); for (int sz = 8; sz <= 2048; sz *= 2) { System.out.printf(" [len=%d]%n", sz); final char[] chars = Arrays.copyOf(ca, sz); final String str = new String(chars); final byte[] bs = str.getBytes(cs); Job[] jobs = { new Job("String decode: csn") { public void work() throws Throwable { for (int i = 0; i < itrs; i++) new String(bs, csn); } }, new Job("String decode: cs") { public void work() throws Throwable { for (int i = 0; i < itrs; i++) new String(bs, cs); } }, new Job("String encode: csn") { public void work() throws Throwable { for (int i = 0; i < itrs; i++) str.getBytes(csn); } }, new Job("String encode: cs") { public void work() throws Throwable { for (int i = 0; i < itrs; i++) str.getBytes(cs); } }, }; sum.add(time(jobs)); } } }
/** @return Candidates map. */ Map<K, Collection<GridCacheDgcLockCandidate>> candidatesMap() { return Collections.unmodifiableMap(map); }
/** @return Collection of keys that did not pass the filter. */ public Collection<IgniteTxKey> filterFailedKeys() { return filterFailedKeys == null ? Collections.<IgniteTxKey>emptyList() : filterFailedKeys; }
/** @return Owned values map. */ public Map<IgniteTxKey, CacheVersionedValue> ownedValues() { return ownedVals == null ? Collections.<IgniteTxKey, CacheVersionedValue>emptyMap() : Collections.unmodifiableMap(ownedVals); }
/** * Gets pending versions that are less than {@link #version()}. * * @return Pending versions. */ public Collection<GridCacheVersion> pending() { return pending == null ? Collections.<GridCacheVersion>emptyList() : pending; }
/** * Base class for a data buffer handling basic byte operations among other things. * * @author Adam Gibson */ public abstract class BaseDataBuffer implements DataBuffer { protected int length; protected int offset; protected int elementSize; protected transient ByteBuffer wrappedBuffer; protected Collection<String> referencing = Collections.synchronizedSet(new HashSet<String>()); protected transient WeakReference<DataBuffer> ref; protected boolean isPersist = false; protected AllocationMode allocationMode; protected double[] doubleData; protected int[] intData; protected float[] floatData; protected AtomicBoolean dirty = new AtomicBoolean(false); /** * @param buf * @param length */ protected BaseDataBuffer(ByteBuf buf, int length, int offset) { this(buf, length); this.offset = offset; } /** * @param buf * @param length */ protected BaseDataBuffer(ByteBuf buf, int length) { allocationMode = Nd4j.alloc; this.wrappedBuffer = buf.nioBuffer(); this.length = length; } /** * @param data * @param copy */ public BaseDataBuffer(float[] data, boolean copy, int offset) { this(data, copy); this.offset = offset; } /** * @param data * @param copy */ public BaseDataBuffer(float[] data, boolean copy) { allocationMode = Nd4j.alloc; if (allocationMode == AllocationMode.HEAP) { if (copy) { floatData = ArrayUtil.copy(data); } else { this.floatData = data; } } else { wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length); wrappedBuffer.order(ByteOrder.nativeOrder()); FloatBuffer buffer = wrappedBuffer.asFloatBuffer(); for (int i = 0; i < data.length; i++) { buffer.put(i, data[i]); } } length = data.length; } /** * @param data * @param copy */ public BaseDataBuffer(double[] data, boolean copy, int offset) { this(data, copy); this.offset = offset; } /** * @param data * @param copy */ public BaseDataBuffer(double[] data, boolean copy) { allocationMode = Nd4j.alloc; if (allocationMode == AllocationMode.HEAP) { if (copy) { doubleData = ArrayUtil.copy(data); } else { this.doubleData = data; } } else { wrappedBuffer = ByteBuffer.allocateDirect(8 * data.length); wrappedBuffer.order(ByteOrder.nativeOrder()); DoubleBuffer buffer = wrappedBuffer.asDoubleBuffer(); for (int i = 0; i < data.length; i++) { buffer.put(i, data[i]); } } length = data.length; } /** * @param data * @param copy */ public BaseDataBuffer(int[] data, boolean copy, int offset) { this(data, copy); this.offset = offset; } /** * @param data * @param copy */ public BaseDataBuffer(int[] data, boolean copy) { allocationMode = Nd4j.alloc; if (allocationMode == AllocationMode.HEAP) { if (copy) intData = ArrayUtil.copy(data); else this.intData = data; } else { wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length); wrappedBuffer.order(ByteOrder.nativeOrder()); IntBuffer buffer = wrappedBuffer.asIntBuffer(); for (int i = 0; i < data.length; i++) { buffer.put(i, data[i]); } } length = data.length; } /** @param data */ public BaseDataBuffer(double[] data) { this(data, Nd4j.copyOnOps); } /** @param data */ public BaseDataBuffer(int[] data) { this(data, Nd4j.copyOnOps); } /** @param data */ public BaseDataBuffer(float[] data) { this(data, Nd4j.copyOnOps); } /** * @param length * @param elementSize */ public BaseDataBuffer(int length, int elementSize, int offset) { this(length, elementSize); this.offset = offset; } /** * @param length * @param elementSize */ public BaseDataBuffer(int length, int elementSize) { allocationMode = Nd4j.alloc; this.length = length; this.elementSize = elementSize; if (allocationMode() == AllocationMode.DIRECT) { // allows for creation of the nio byte buffer to be overridden setNioBuffer(); } else if (dataType() == Type.DOUBLE) { doubleData = new double[length]; } else if (dataType() == Type.FLOAT) { floatData = new float[length]; } else if (dataType() == Type.INT) intData = new int[length]; } /** * Create a data buffer from the given length * * @param buffer * @param length */ public BaseDataBuffer(ByteBuffer buffer, int length, int offset) { this(buffer, length); this.offset = offset; } /** * Create a data buffer from the given length * * @param buffer * @param length */ public BaseDataBuffer(ByteBuffer buffer, int length) { allocationMode = Nd4j.alloc; this.length = length; buffer.order(ByteOrder.nativeOrder()); if (allocationMode() == AllocationMode.DIRECT) { this.wrappedBuffer = buffer; } else if (dataType() == Type.INT) { intData = new int[length]; IntBuffer intBuffer = buffer.asIntBuffer(); for (int i = 0; i < length; i++) { intData[i] = intBuffer.get(i); } } else if (dataType() == Type.DOUBLE) { doubleData = new double[length]; DoubleBuffer doubleBuffer = buffer.asDoubleBuffer(); for (int i = 0; i < length; i++) { doubleData[i] = doubleBuffer.get(i); } } else if (dataType() == Type.FLOAT) { floatData = new float[length]; FloatBuffer floatBuffer = buffer.asFloatBuffer(); for (int i = 0; i < length; i++) { floatData[i] = floatBuffer.get(i); } } } // sets the nio wrapped buffer (allows to be overridden for other use cases like cuda) protected void setNioBuffer() { wrappedBuffer = ByteBuffer.allocateDirect(elementSize * length); wrappedBuffer.order(ByteOrder.nativeOrder()); } public BaseDataBuffer(byte[] data, int length) { this(Unpooled.wrappedBuffer(data), length); } @Override public int offset() { return offset; } @Override public AllocationMode allocationMode() { return allocationMode; } @Override public void persist() { isPersist = true; } @Override public boolean isPersist() { return isPersist; } @Override public void unPersist() { isPersist = false; } /** * Instantiate a buffer with the given length * * @param length the length of the buffer */ protected BaseDataBuffer(int length) { this.length = length; allocationMode = Nd4j.alloc; if (length < 0) throw new IllegalArgumentException("Unable to create a buffer of length <= 0"); ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue()); if (allocationMode == AllocationMode.HEAP) { if (length >= Integer.MAX_VALUE) throw new IllegalArgumentException( "Length of data buffer can not be > Integer.MAX_VALUE for heap (array based storage) allocation"); if (dataType() == Type.DOUBLE) doubleData = new double[length]; else if (dataType() == Type.FLOAT) floatData = new float[length]; } else { if (length * getElementSize() < 0) throw new IllegalArgumentException( "Unable to create buffer of length " + length + " due to negative length specified"); wrappedBuffer = ByteBuffer.allocateDirect(getElementSize() * length).order(ByteOrder.nativeOrder()); } } @Override public void copyAtStride( DataBuffer buf, int n, int stride, int yStride, int offset, int yOffset) { if (dataType() == Type.FLOAT) { for (int i = 0; i < n; i++) { put(offset + i * stride, buf.getFloat(yOffset + i * yStride)); } } else { for (int i = 0; i < n; i++) { put(offset + i * stride, buf.getDouble(yOffset + i * yStride)); } } } @Override public void removeReferencing(String id) { referencing.remove(id); } @Override public Collection<String> references() { return referencing; } @Override public void addReferencing(String id) { referencing.add(id); } @Override public void assign(int[] indices, float[] data, boolean contiguous, int inc) { if (indices.length != data.length) throw new IllegalArgumentException("Indices and data length must be the same"); if (indices.length > length()) throw new IllegalArgumentException( "More elements than space to assign. This buffer is of length " + length() + " where the indices are of length " + data.length); for (int i = 0; i < indices.length; i++) { put(indices[i], data[i]); } } @Override public void setData(int[] data) { if (intData != null) this.intData = data; else { for (int i = 0; i < data.length; i++) { put(i, data[i]); } } } @Override public void setData(float[] data) { if (floatData != null) { this.floatData = data; } else { for (int i = 0; i < data.length; i++) put(i, data[i]); } } @Override public void setData(double[] data) { if (doubleData != null) { this.doubleData = data; } else { for (int i = 0; i < data.length; i++) put(i, data[i]); } } @Override public void assign(int[] indices, double[] data, boolean contiguous, int inc) { if (indices.length != data.length) throw new IllegalArgumentException("Indices and data length must be the same"); if (indices.length > length()) throw new IllegalArgumentException( "More elements than space to assign. This buffer is of length " + length() + " where the indices are of length " + data.length); for (int i = 0; i < indices.length; i += inc) { put(indices[i], data[i]); } } @Override public void assign(DataBuffer data) { if (data.length() != length()) throw new IllegalArgumentException( "Unable to assign buffer of length " + data.length() + " to this buffer of length " + length()); for (int i = 0; i < data.length(); i++) { put(i, data.getDouble(i)); } } @Override public void assign(int[] indices, float[] data, boolean contiguous) { assign(indices, data, contiguous, 1); } @Override public void assign(int[] indices, double[] data, boolean contiguous) { assign(indices, data, contiguous, 1); } @Override public int length() { return length; } @Override public void assign(Number value) { for (int i = 0; i < length(); i++) assign(value, i); } @Override public double[] getDoublesAt(int offset, int length) { return getDoublesAt(offset, 1, length); } @Override public float[] getFloatsAt(int offset, int inc, int length) { if (offset + length > length()) length -= offset; float[] ret = new float[length]; for (int i = 0; i < length; i++) { ret[i] = getFloat(i + offset); } return ret; } @Override public DataBuffer dup() { if (floatData != null) { return create(floatData); } else if (doubleData != null) { return create(doubleData); } else if (intData != null) { return create(intData); } DataBuffer ret = create(length); for (int i = 0; i < ret.length(); i++) ret.put(i, getDouble(i)); return ret; } /** * Create with length * * @param length a databuffer of the same type as this with the given length * @return a data buffer with the same length and datatype as this one */ protected abstract DataBuffer create(int length); /** * Create the data buffer with respect to the given byte buffer * * @param data the buffer to create * @return the data buffer based on the given buffer */ public abstract DataBuffer create(double[] data); /** * Create the data buffer with respect to the given byte buffer * * @param data the buffer to create * @return the data buffer based on the given buffer */ public abstract DataBuffer create(float[] data); /** * Create the data buffer with respect to the given byte buffer * * @param data the buffer to create * @return the data buffer based on the given buffer */ public abstract DataBuffer create(int[] data); /** * Create the data buffer with respect to the given byte buffer * * @param buf the buffer to create * @return the data buffer based on the given buffer */ public abstract DataBuffer create(ByteBuf buf, int length); @Override public double[] getDoublesAt(int offset, int inc, int length) { if (offset + length > length()) length -= offset; double[] ret = new double[length]; for (int i = 0; i < length; i++) { ret[i] = getDouble(i + offset); } return ret; } @Override public float[] getFloatsAt(int offset, int length) { return getFloatsAt(offset, 1, length); } @Override public IComplexFloat getComplexFloat(int i) { return Nd4j.createFloat(getFloat(i), getFloat(i + 1)); } @Override public IComplexDouble getComplexDouble(int i) { return Nd4j.createDouble(getDouble(i), getDouble(i + 1)); } @Override public IComplexNumber getComplex(int i) { return dataType() == DataBuffer.Type.FLOAT ? getComplexFloat(i) : getComplexDouble(i); } @Override public void put(int i, IComplexNumber result) { put(i, result.realComponent().doubleValue()); put(i + 1, result.imaginaryComponent().doubleValue()); } @Override public void assign(int[] offsets, int[] strides, DataBuffer... buffers) { assign(offsets, strides, length(), buffers); } @Override public byte[] asBytes() { if (allocationMode == AllocationMode.HEAP) { ByteArrayOutputStream bos = new ByteArrayOutputStream(getElementSize() * length()); DataOutputStream dos = new DataOutputStream(bos); if (dataType() == Type.DOUBLE) { if (doubleData == null) throw new IllegalStateException("Double array is null!"); try { for (int i = 0; i < doubleData.length; i++) dos.writeDouble(doubleData[i]); } catch (IOException e) { throw new RuntimeException(e); } } else { if (floatData == null) throw new IllegalStateException("Double array is null!"); try { for (int i = 0; i < floatData.length; i++) dos.writeFloat(floatData[i]); } catch (IOException e) { throw new RuntimeException(e); } } return bos.toByteArray(); } else { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bos); if (dataType() == Type.DOUBLE) { for (int i = 0; i < length(); i++) { try { dos.writeDouble(getDouble(i)); } catch (IOException e) { e.printStackTrace(); } } } else { for (int i = 0; i < length(); i++) { try { dos.writeFloat(getFloat(i)); } catch (IOException e) { e.printStackTrace(); } } } return bos.toByteArray(); } } @Override public float[] asFloat() { if (allocationMode == AllocationMode.HEAP) { if (floatData != null) { return floatData; } } float[] ret = new float[length]; for (int i = 0; i < length; i++) ret[i] = getFloat(i); return ret; } @Override public double[] asDouble() { if (allocationMode == AllocationMode.HEAP) { if (doubleData != null) { return doubleData; } } double[] ret = new double[length]; for (int i = 0; i < length; i++) ret[i] = getDouble(i); return ret; } @Override public int[] asInt() { if (allocationMode == AllocationMode.HEAP) { if (intData != null) { return intData; } } return wrappedBuffer.asIntBuffer().array(); } @Override public double getDouble(int i) { if (doubleData != null) { if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i); dirty.set(false); return doubleData[i]; } else if (floatData != null) { if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i); dirty.set(false); return (double) floatData[i]; } else if (intData != null) { dirty.set(false); return (double) intData[i]; } if (dataType() == Type.FLOAT) { dirty.set(false); return wrappedBuffer.asFloatBuffer().get(i); } else if (dataType() == Type.INT) { dirty.set(false); return wrappedBuffer.asIntBuffer().get(i); } else { dirty.set(false); return wrappedBuffer.asDoubleBuffer().get(i); } } @Override public float getFloat(int i) { if (doubleData != null) { if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i); dirty.set(false); return (float) doubleData[i]; } else if (floatData != null) { if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i); dirty.set(false); return floatData[i]; } else if (intData != null) { dirty.set(false); return (float) intData[i]; } if (dataType() == Type.DOUBLE) { dirty.set(false); return (float) wrappedBuffer.asDoubleBuffer().get(i); } dirty.getAndSet(true); return wrappedBuffer.asFloatBuffer().get(i); } @Override public Number getNumber(int i) { if (dataType() == Type.DOUBLE) return getDouble(i); else if (dataType() == Type.INT) return getInt(i); return getFloat(i); } @Override public void put(int i, float element) { put(i, (double) element); } @Override public void put(int i, double element) { if (i < 0 || i >= length()) throw new IllegalArgumentException("Illegal index " + i); if (doubleData != null) doubleData[i] = element; else if (floatData != null) floatData[i] = (float) element; else if (intData != null) intData[i] = (int) element; else { if (dataType() == Type.DOUBLE) { wrappedBuffer.asDoubleBuffer().put(i, element); } else if (dataType() == Type.INT) { wrappedBuffer.asIntBuffer().put(i, (int) element); } else { wrappedBuffer.asFloatBuffer().put(i, (float) element); } } dirty.set(true); } @Override public boolean dirty() { return dirty.get(); } @Override public IntBuffer asNioInt() { if (wrappedBuffer == null) { return IntBuffer.wrap(intData); } return wrappedBuffer.asIntBuffer(); } @Override public DoubleBuffer asNioDouble() { if (wrappedBuffer == null) { return DoubleBuffer.wrap(doubleData); } return wrappedBuffer.asDoubleBuffer(); } @Override public FloatBuffer asNioFloat() { if (wrappedBuffer == null) { return FloatBuffer.wrap(floatData); } return wrappedBuffer.asFloatBuffer(); } @Override public ByteBuffer asNio() { return wrappedBuffer; } @Override public ByteBuf asNetty() { if (wrappedBuffer != null) return Unpooled.wrappedBuffer(wrappedBuffer); else if (floatData != null) return Unpooled.copyFloat(floatData); else if (doubleData != null) return Unpooled.copyDouble(doubleData); throw new IllegalStateException("No data source defined"); } @Override public void put(int i, int element) { put(i, (double) element); } @Override public void assign(Number value, int offset) { for (int i = offset; i < length(); i++) put(i, value.doubleValue()); } @Override public void write(OutputStream dos) { if (dos instanceof DataOutputStream) { try { write((DataOutputStream) dos); } catch (IOException e) { e.printStackTrace(); } } else { DataOutputStream dos2 = new DataOutputStream(dos); try { write(dos2); } catch (IOException e) { e.printStackTrace(); } } } @Override public void read(InputStream is) { if (is instanceof DataInputStream) { read((DataInputStream) is); } else { DataInputStream dis2 = new DataInputStream(is); read(dis2); } } @Override public void flush() {} @Override public int getInt(int ix) { return (int) getDouble(ix); } @Override public void assign(int[] offsets, int[] strides, long n, DataBuffer... buffers) { if (offsets.length != strides.length || strides.length != buffers.length) throw new IllegalArgumentException( "Unable to assign buffers, please specify equal lengths strides, offsets, and buffers"); int length = 0; for (int i = 0; i < buffers.length; i++) length += buffers[i].length(); int count = 0; for (int i = 0; i < buffers.length; i++) { for (int j = offsets[i]; j < buffers[i].length(); j += strides[i]) { put(count++, buffers[i].getDouble(j)); } } if (count != n) throw new IllegalArgumentException("Strides and offsets didn't match up to length " + n); } @Override public void assign(DataBuffer... buffers) { int[] offsets = new int[buffers.length]; int[] strides = new int[buffers.length]; for (int i = 0; i < strides.length; i++) strides[i] = 1; assign(offsets, strides, buffers); } @Override public void destroy() {} @Override public boolean equals(Object o) { if (o instanceof DataBuffer) { DataBuffer d = (DataBuffer) o; if (d.length() != length()) return false; for (int i = 0; i < length(); i++) { double eps = Math.abs(getDouble(i) - d.getDouble(i)); if (eps > Nd4j.EPS_THRESHOLD) return false; } } return true; } private void readObject(ObjectInputStream s) { doReadObject(s); } private void writeObject(java.io.ObjectOutputStream out) throws IOException { out.defaultWriteObject(); write(out); } protected void doReadObject(ObjectInputStream s) { try { s.defaultReadObject(); read(s); } catch (Exception e) { throw new RuntimeException(e); } } protected void read(DataInputStream s) { try { ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue()); referencing = Collections.synchronizedSet(new HashSet<String>()); dirty = new AtomicBoolean(false); allocationMode = AllocationMode.valueOf(s.readUTF()); length = s.readInt(); Type t = Type.valueOf(s.readUTF()); if (t == Type.DOUBLE) { if (allocationMode == AllocationMode.HEAP) { if (this.dataType() == Type.FLOAT) { // DataBuffer type // double -> float floatData = new float[length()]; } else if (this.dataType() == Type.DOUBLE) { // double -> double doubleData = new double[length()]; } else { // double -> int intData = new int[length()]; } for (int i = 0; i < length(); i++) { put(i, s.readDouble()); } } else { wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize()); wrappedBuffer.order(ByteOrder.nativeOrder()); for (int i = 0; i < length(); i++) { put(i, s.readDouble()); } } } else { if (allocationMode == AllocationMode.HEAP) { if (this.dataType() == Type.FLOAT) { // DataBuffer type // float -> float floatData = new float[length()]; } else if (this.dataType() == Type.DOUBLE) { // float -> double doubleData = new double[length()]; } else { // float-> int intData = new int[length()]; } for (int i = 0; i < length(); i++) { put(i, s.readFloat()); } } else { wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize()); wrappedBuffer.order(ByteOrder.nativeOrder()); for (int i = 0; i < length(); i++) { put(i, s.readFloat()); } } } } catch (Exception e) { throw new RuntimeException(e); } } protected void write(DataOutputStream out) throws IOException { out.writeUTF(allocationMode.name()); out.writeInt(length()); out.writeUTF(dataType().name()); if (dataType() == Type.DOUBLE) { for (int i = 0; i < length(); i++) out.writeDouble(getDouble(i)); } else { for (int i = 0; i < length(); i++) out.writeFloat(getFloat(i)); } } @Override public Object array() { if (floatData != null) return floatData; if (doubleData != null) return doubleData; throw new UnsupportedOperationException(); } @Override public String toString() { StringBuffer ret = new StringBuffer(); ret.append("["); for (int i = 0; i < length(); i++) { ret.append(getNumber(i)); if (i < length() - 1) ret.append(","); } ret.append("]"); return ret.toString(); } @Override public int hashCode() { int result = length; result = 31 * result + (referencing != null ? referencing.hashCode() : 0); result = 31 * result + (ref != null ? ref.hashCode() : 0); result = 31 * result + (isPersist ? 1 : 0); result = 31 * result + (allocationMode != null ? allocationMode.hashCode() : 0); return result; } }
/** @return Map of versions to be verified. */ public Map<IgniteTxKey, GridCacheVersion> dhtVersions() { return dhtVers == null ? Collections.<IgniteTxKey, GridCacheVersion>emptyMap() : dhtVers; }
/** * Grabs local events and detects if events was lost since last poll. * * @param ignite Target grid. * @param evtOrderKey Unique key to take last order key from node local map. * @param evtThrottleCntrKey Unique key to take throttle count from node local map. * @param evtTypes Event types to collect. * @param evtMapper Closure to map grid events to Visor data transfer objects. * @return Collections of node events */ public static Collection<VisorGridEvent> collectEvents( Ignite ignite, String evtOrderKey, String evtThrottleCntrKey, final int[] evtTypes, IgniteClosure<Event, VisorGridEvent> evtMapper) { assert ignite != null; assert evtTypes != null && evtTypes.length > 0; ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap(); final long lastOrder = getOrElse(nl, evtOrderKey, -1L); final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L); // When we first time arrive onto a node to get its local events, // we'll grab only last those events that not older than given period to make sure we are // not grabbing GBs of data accidentally. final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW; // Flag for detecting gaps between events. final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0); IgnitePredicate<Event> p = new IgnitePredicate<Event>() { /** */ private static final long serialVersionUID = 0L; @Override public boolean apply(Event e) { // Detects that events were lost. if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true); // Retains events by lastOrder, period and type. return e.localOrder() > lastOrder && e.timestamp() > notOlderThan && F.contains(evtTypes, e.type()); } }; Collection<Event> evts = ignite.events().localQuery(p); // Update latest order in node local, if not empty. if (!evts.isEmpty()) { Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR); nl.put(evtOrderKey, maxEvt.localOrder()); } // Update throttle counter. if (!lastFound.get()) nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1); boolean lost = !lastFound.get() && throttle == 0; Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0)); if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id())); for (Event e : evts) { VisorGridEvent visorEvt = evtMapper.apply(e); if (visorEvt != null) res.add(visorEvt); } return res; }