void _pickInitial() throws MongoException {
if (_curAddress != null) return;
// we need to just get a server to query for ismaster
_pickCurrent();
try {
_logger.info("current address beginning of _pickInitial: " + _curAddress);
DBObject im = isMasterCmd();
if (_isMaster(im)) return;
synchronized (_allHosts) {
Collections.shuffle(_allHosts);
for (ServerAddress a : _allHosts) {
if (_curAddress == a) continue;
_logger.info("remote [" + _curAddress + "] -> [" + a + "]");
_set(a);
im = isMasterCmd();
if (_isMaster(im)) return;
_logger.severe("switched to: " + a + " but isn't master");
}
throw new MongoException("can't find master");
}
} catch (Exception e) {
_logger.log(Level.SEVERE, "can't pick initial master, using random one", e);
}
}
protected void read(DataInputStream s) {
try {
ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
referencing = Collections.synchronizedSet(new HashSet<String>());
dirty = new AtomicBoolean(false);
allocationMode = AllocationMode.valueOf(s.readUTF());
length = s.readInt();
Type t = Type.valueOf(s.readUTF());
if (t == Type.DOUBLE) {
if (allocationMode == AllocationMode.HEAP) {
if (this.dataType() == Type.FLOAT) { // DataBuffer type
// double -> float
floatData = new float[length()];
} else if (this.dataType() == Type.DOUBLE) {
// double -> double
doubleData = new double[length()];
} else {
// double -> int
intData = new int[length()];
}
for (int i = 0; i < length(); i++) {
put(i, s.readDouble());
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
wrappedBuffer.order(ByteOrder.nativeOrder());
for (int i = 0; i < length(); i++) {
put(i, s.readDouble());
}
}
} else {
if (allocationMode == AllocationMode.HEAP) {
if (this.dataType() == Type.FLOAT) { // DataBuffer type
// float -> float
floatData = new float[length()];
} else if (this.dataType() == Type.DOUBLE) {
// float -> double
doubleData = new double[length()];
} else {
// float-> int
intData = new int[length()];
}
for (int i = 0; i < length(); i++) {
put(i, s.readFloat());
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
wrappedBuffer.order(ByteOrder.nativeOrder());
for (int i = 0; i < length(); i++) {
put(i, s.readFloat());
}
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
static void testMixed(Charset cs) throws Throwable {
CharsetDecoder dec =
cs.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
CharsetEncoder enc =
cs.newEncoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
List<Integer> cps = new ArrayList<>(0x10000);
int off = 0;
int cp = 0;
while (cp < 0x10000) {
if (enc.canEncode((char) cp)) {
cps.add(cp);
}
cp++;
}
Collections.shuffle(cps);
char[] bmpCA = new char[cps.size()];
for (int i = 0; i < cps.size(); i++) bmpCA[i] = (char) (int) cps.get(i);
String bmpStr = new String(bmpCA);
// getBytes(csn);
byte[] bmpBA = bmpStr.getBytes(cs.name());
ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(bmpCA));
byte[] baNIO = new byte[bf.limit()];
bf.get(baNIO, 0, baNIO.length);
if (!Arrays.equals(bmpBA, baNIO)) {
throw new RuntimeException("getBytes(csn) failed -> " + cs.name());
}
// getBytes(cs);
bmpBA = bmpStr.getBytes(cs);
if (!Arrays.equals(bmpBA, baNIO))
throw new RuntimeException("getBytes(cs) failed -> " + cs.name());
// new String(csn);
String strSC = new String(bmpBA, cs.name());
String strNIO = dec.reset().decode(ByteBuffer.wrap(bmpBA)).toString();
if (!strNIO.equals(strSC)) {
throw new RuntimeException("new String(csn) failed -> " + cs.name());
}
// new String(cs);
strSC = new String(bmpBA, cs);
if (!strNIO.equals(strSC)) throw new RuntimeException("new String(cs) failed -> " + cs.name());
}
/**
* @param fld Folder with files to match.
* @param ptrn Pattern to match against file name.
* @return Collection of matched files.
*/
public static List<VisorLogFile> matchedFiles(File fld, final String ptrn) {
List<VisorLogFile> files =
fileTree(
fld,
MAX_FOLDER_DEPTH,
new FileFilter() {
@Override
public boolean accept(File f) {
return !f.isHidden()
&& (f.isDirectory() || f.isFile() && f.getName().matches(ptrn));
}
});
Collections.sort(files, LAST_MODIFIED);
return files;
}
/**
* Selects the best text <code>DataFlavor</code> from an array of <code>
* DataFlavor</code>s. Only <code>DataFlavor.stringFlavor</code>, and equivalent flavors, and
* flavors that have a primary MIME type of "text", are considered for selection.
*
* <p>Flavors are first sorted by their MIME types in the following order:
*
* <ul>
* <li>"text/sgml"
* <li>"text/xml"
* <li>"text/html"
* <li>"text/rtf"
* <li>"text/enriched"
* <li>"text/richtext"
* <li>"text/uri-list"
* <li>"text/tab-separated-values"
* <li>"text/t140"
* <li>"text/rfc822-headers"
* <li>"text/parityfec"
* <li>"text/directory"
* <li>"text/css"
* <li>"text/calendar"
* <li>"application/x-java-serialized-object"
* <li>"text/plain"
* <li>"text/<other>"
* </ul>
*
* <p>For example, "text/sgml" will be selected over "text/html", and <code>
* DataFlavor.stringFlavor</code> will be chosen over <code>DataFlavor.plainTextFlavor</code>.
*
* <p>If two or more flavors share the best MIME type in the array, then that MIME type will be
* checked to see if it supports the charset parameter.
*
* <p>The following MIME types support, or are treated as though they support, the charset
* parameter:
*
* <ul>
* <li>"text/sgml"
* <li>"text/xml"
* <li>"text/html"
* <li>"text/enriched"
* <li>"text/richtext"
* <li>"text/uri-list"
* <li>"text/directory"
* <li>"text/css"
* <li>"text/calendar"
* <li>"application/x-java-serialized-object"
* <li>"text/plain"
* </ul>
*
* The following MIME types do not support, or are treated as though they do not support, the
* charset parameter:
*
* <ul>
* <li>"text/rtf"
* <li>"text/tab-separated-values"
* <li>"text/t140"
* <li>"text/rfc822-headers"
* <li>"text/parityfec"
* </ul>
*
* For "text/<other>" MIME types, the first time the JRE needs to determine whether the MIME
* type supports the charset parameter, it will check whether the parameter is explicitly listed
* in an arbitrarily chosen <code>DataFlavor</code> which uses that MIME type. If so, the JRE will
* assume from that point on that the MIME type supports the charset parameter and will not check
* again. If the parameter is not explicitly listed, the JRE will assume from that point on that
* the MIME type does not support the charset parameter and will not check again. Because this
* check is performed on an arbitrarily chosen <code>DataFlavor</code>, developers must ensure
* that all <code>DataFlavor</code>s with a "text/<other>" MIME type specify the charset
* parameter if it is supported by that MIME type. Developers should never rely on the JRE to
* substitute the platform's default charset for a "text/<other>" DataFlavor. Failure to
* adhere to this restriction will lead to undefined behavior.
*
* <p>If the best MIME type in the array does not support the charset parameter, the flavors which
* share that MIME type will then be sorted by their representation classes in the following
* order: <code>java.io.InputStream</code>, <code>java.nio.ByteBuffer</code>, <code>[B</code>,
* <all others>.
*
* <p>If two or more flavors share the best representation class, or if no flavor has one of the
* three specified representations, then one of those flavors will be chosen
* non-deterministically.
*
* <p>If the best MIME type in the array does support the charset parameter, the flavors which
* share that MIME type will then be sorted by their representation classes in the following
* order: <code>java.io.Reader</code>, <code>java.lang.String</code>, <code>java.nio.CharBuffer
* </code>, <code>[C</code>, <all others>.
*
* <p>If two or more flavors share the best representation class, and that representation is one
* of the four explicitly listed, then one of those flavors will be chosen non-deterministically.
* If, however, no flavor has one of the four specified representations, the flavors will then be
* sorted by their charsets. Unicode charsets, such as "UTF-16", "UTF-8", "UTF-16BE", "UTF-16LE",
* and their aliases, are considered best. After them, the platform default charset and its
* aliases are selected. "US-ASCII" and its aliases are worst. All other charsets are chosen in
* alphabetical order, but only charsets supported by this implementation of the Java platform
* will be considered.
*
* <p>If two or more flavors share the best charset, the flavors will then again be sorted by
* their representation classes in the following order: <code>java.io.InputStream</code>, <code>
* java.nio.ByteBuffer</code>, <code>[B</code>, <all others>.
*
* <p>If two or more flavors share the best representation class, or if no flavor has one of the
* three specified representations, then one of those flavors will be chosen
* non-deterministically.
*
* @param availableFlavors an array of available <code>DataFlavor</code>s
* @return the best (highest fidelity) flavor according to the rules specified above, or <code>
* null</code>, if <code>availableFlavors</code> is <code>null</code>, has zero length, or
* contains no text flavors
* @since 1.3
*/
public static final DataFlavor selectBestTextFlavor(DataFlavor[] availableFlavors) {
if (availableFlavors == null || availableFlavors.length == 0) {
return null;
}
if (textFlavorComparator == null) {
textFlavorComparator = new TextFlavorComparator();
}
DataFlavor bestFlavor =
(DataFlavor) Collections.max(Arrays.asList(availableFlavors), textFlavorComparator);
if (!bestFlavor.isFlavorTextType()) {
return null;
}
return bestFlavor;
}
/**
* Finds all files in folder and in it's sub-tree of specified depth.
*
* @param file Starting folder
* @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
* @param filter file filter.
* @return List of found files.
*/
public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
if (file.isDirectory()) {
File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
if (files == null) return Collections.emptyList();
List<VisorLogFile> res = new ArrayList<>(files.length);
for (File f : files) {
if (f.isFile() && f.length() > 0) res.add(new VisorLogFile(f));
else if (maxDepth > 1) res.addAll(fileTree(f, maxDepth - 1, filter));
}
return res;
}
return F.asList(new VisorLogFile(file));
}
private void _pickCurrent() throws MongoException {
if (_allHosts == null)
throw new MongoException(
"got master/slave issue but not in master/slave mode on the client side");
synchronized (_allHosts) {
Collections.shuffle(_allHosts);
for (int i = 0; i < _allHosts.size(); i++) {
ServerAddress a = _allHosts.get(i);
if (a == _curAddress) continue;
if (_curAddress != null) {
_logger.info("switching from [" + _curAddress + "] to [" + a + "]");
}
_set(a);
return;
}
}
throw new MongoException("couldn't find a new host to swtich too");
}
/** @return Near keys. */
public List<byte[]> nearKeyBytes() {
return nearKeyBytes != null ? nearKeyBytes : Collections.<byte[]>emptyList();
}
public class DBPort {
public static final int PORT = 27017;
static final boolean USE_NAGLE = false;
static final long CONN_RETRY_TIME_MS = 15000;
public DBPort(InetSocketAddress addr) throws IOException {
this(addr, null, new MongoOptions());
}
DBPort(InetSocketAddress addr, DBPortPool pool, MongoOptions options) throws IOException {
_options = options;
_addr = addr;
_pool = pool;
_hashCode = _addr.hashCode();
_logger = Logger.getLogger(_rootLogger.getName() + "." + addr.toString());
}
/** @param response will get wiped */
DBMessage call(DBMessage msg, ByteDecoder decoder) throws IOException {
return go(msg, decoder);
}
void say(DBMessage msg) throws IOException {
go(msg, null);
}
private synchronized DBMessage go(DBMessage msg, ByteDecoder decoder) throws IOException {
if (_sock == null) _open();
{
ByteBuffer out = msg.prepare();
while (out.remaining() > 0) _sock.write(out);
}
if (_pool != null) _pool._everWorked = true;
if (decoder == null) return null;
ByteBuffer response = decoder._buf;
if (response.position() != 0) throw new IllegalArgumentException();
int read = 0;
while (read < DBMessage.HEADER_LENGTH) read += _read(response);
int len = response.getInt(0);
if (len <= DBMessage.HEADER_LENGTH)
throw new IllegalArgumentException("db sent invalid length: " + len);
if (len > response.capacity())
throw new IllegalArgumentException(
"db message size is too big (" + len + ") " + "max is (" + response.capacity() + ")");
response.limit(len);
while (read < len) read += _read(response);
if (read != len) throw new RuntimeException("something is wrong");
response.flip();
return new DBMessage(response);
}
public synchronized void ensureOpen() throws IOException {
if (_sock != null) return;
_open();
}
void _open() throws IOException {
long sleepTime = 100;
final long start = System.currentTimeMillis();
while (true) {
IOException lastError = null;
try {
_sock = SocketChannel.open();
_socket = _sock.socket();
_socket.connect(_addr, _options.connectTimeout);
_socket.setTcpNoDelay(!USE_NAGLE);
_socket.setSoTimeout(_options.socketTimeout);
_in = _socket.getInputStream();
return;
} catch (IOException ioe) {
// TODO - erh to fix lastError = new IOException( "couldn't connect to [" +
// _addr + "] bc:" + lastError , lastError );
lastError = new IOException("couldn't connect to [" + _addr + "] bc:" + ioe);
_logger.log(Level.INFO, "connect fail to : " + _addr, ioe);
}
if (!_options.autoConnectRetry || (_pool != null && !_pool._everWorked)) throw lastError;
long sleptSoFar = System.currentTimeMillis() - start;
if (sleptSoFar >= CONN_RETRY_TIME_MS) throw lastError;
if (sleepTime + sleptSoFar > CONN_RETRY_TIME_MS) sleepTime = CONN_RETRY_TIME_MS - sleptSoFar;
_logger.severe(
"going to sleep and retry. total sleep time after = "
+ (sleptSoFar + sleptSoFar)
+ "ms this time:"
+ sleepTime
+ "ms");
ThreadUtil.sleep(sleepTime);
sleepTime *= 2;
}
}
public int hashCode() {
return _hashCode;
}
public String host() {
return _addr.toString();
}
public String toString() {
return "{DBPort " + host() + "}";
}
protected void finalize() {
if (_sock != null) {
try {
_sock.close();
} catch (Exception e) {
// don't care
}
_in = null;
_socket = null;
_sock = null;
}
}
void checkAuth(DB db) {
if (db._username == null) return;
if (_authed.containsKey(db)) return;
if (_inauth) return;
_inauth = true;
try {
if (db.reauth()) {
_authed.put(db, true);
return;
}
} finally {
_inauth = false;
}
throw new MongoInternalException("can't reauth!");
}
private int _read(ByteBuffer buf) throws IOException {
int x = _in.read(buf.array(), buf.position(), buf.remaining());
if (x < 0) throw new IOException("connection to server closed unexpectedly");
buf.position(buf.position() + x);
return x;
}
final int _hashCode;
final InetSocketAddress _addr;
final DBPortPool _pool;
final MongoOptions _options;
final Logger _logger;
private SocketChannel _sock;
private Socket _socket;
private InputStream _in;
private boolean _inauth = false;
private Map<DB, Boolean> _authed = Collections.synchronizedMap(new WeakHashMap<DB, Boolean>());
private static Logger _rootLogger = Logger.getLogger("com.mongodb.port");
}
public static void main(String[] args) throws Throwable {
final int itrs = Integer.getInteger("iterations", 100000);
// final int itrs = Integer.getInteger("iterations", 12);
final int size = Integer.getInteger("size", 2048);
final int subsize = Integer.getInteger("subsize", 128);
final int maxchar = Integer.getInteger("maxchar", 128);
final String regex = System.getProperty("filter");
final Pattern filter = (regex == null) ? null : Pattern.compile(regex);
final boolean useSecurityManager = Boolean.getBoolean("SecurityManager");
if (useSecurityManager) System.setSecurityManager(new PermissiveSecurityManger());
final Random rnd = new Random();
String[] csns =
new String[] {
"Big5",
"Johab",
"EUC_CN",
"EUC_KR",
"MS932",
"MS936",
"MS949",
"MS950",
"GBK",
"Big5_HKSCS",
"Big5_HKSCS_2001",
"Big5_Solaris",
"MS950_HKSCS",
"MS950_HKSCS_XP",
"IBM1364",
"IBM1381",
"IBM1383",
"IBM930",
"IBM933",
"IBM935",
"IBM937",
"IBM939",
"IBM942",
"IBM943",
"IBM948",
"IBM949",
"IBM950",
"IBM970",
};
ArrayList<long[]> sum = new ArrayList<>();
for (final String csn : csns) {
final Charset cs = Charset.forName(csn);
List<Integer> cps = new ArrayList<>(0x4000);
int off = 0;
int cp = 0;
int n = 0;
CharsetEncoder enc = cs.newEncoder();
while (cp < 0x10000 && n < cps.size()) {
if (enc.canEncode((char) cp)) {
cps.add(cp);
n++;
}
cp++;
}
Collections.shuffle(cps);
char[] ca = new char[cps.size()];
for (int i = 0; i < cps.size(); i++) ca[i] = (char) (int) cps.get(i);
System.out.printf("%n--------%s---------%n", csn);
for (int sz = 8; sz <= 2048; sz *= 2) {
System.out.printf(" [len=%d]%n", sz);
final char[] chars = Arrays.copyOf(ca, sz);
final String str = new String(chars);
final byte[] bs = str.getBytes(cs);
Job[] jobs = {
new Job("String decode: csn") {
public void work() throws Throwable {
for (int i = 0; i < itrs; i++) new String(bs, csn);
}
},
new Job("String decode: cs") {
public void work() throws Throwable {
for (int i = 0; i < itrs; i++) new String(bs, cs);
}
},
new Job("String encode: csn") {
public void work() throws Throwable {
for (int i = 0; i < itrs; i++) str.getBytes(csn);
}
},
new Job("String encode: cs") {
public void work() throws Throwable {
for (int i = 0; i < itrs; i++) str.getBytes(cs);
}
},
};
sum.add(time(jobs));
}
}
}
/** @return Candidates map. */
Map<K, Collection<GridCacheDgcLockCandidate>> candidatesMap() {
return Collections.unmodifiableMap(map);
}
/** @return Collection of keys that did not pass the filter. */
public Collection<IgniteTxKey> filterFailedKeys() {
return filterFailedKeys == null ? Collections.<IgniteTxKey>emptyList() : filterFailedKeys;
}
/** @return Owned values map. */
public Map<IgniteTxKey, CacheVersionedValue> ownedValues() {
return ownedVals == null
? Collections.<IgniteTxKey, CacheVersionedValue>emptyMap()
: Collections.unmodifiableMap(ownedVals);
}
/**
* Gets pending versions that are less than {@link #version()}.
*
* @return Pending versions.
*/
public Collection<GridCacheVersion> pending() {
return pending == null ? Collections.<GridCacheVersion>emptyList() : pending;
}
/**
* Base class for a data buffer handling basic byte operations among other things.
*
* @author Adam Gibson
*/
public abstract class BaseDataBuffer implements DataBuffer {
protected int length;
protected int offset;
protected int elementSize;
protected transient ByteBuffer wrappedBuffer;
protected Collection<String> referencing = Collections.synchronizedSet(new HashSet<String>());
protected transient WeakReference<DataBuffer> ref;
protected boolean isPersist = false;
protected AllocationMode allocationMode;
protected double[] doubleData;
protected int[] intData;
protected float[] floatData;
protected AtomicBoolean dirty = new AtomicBoolean(false);
/**
* @param buf
* @param length
*/
protected BaseDataBuffer(ByteBuf buf, int length, int offset) {
this(buf, length);
this.offset = offset;
}
/**
* @param buf
* @param length
*/
protected BaseDataBuffer(ByteBuf buf, int length) {
allocationMode = Nd4j.alloc;
this.wrappedBuffer = buf.nioBuffer();
this.length = length;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(float[] data, boolean copy, int offset) {
this(data, copy);
this.offset = offset;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(float[] data, boolean copy) {
allocationMode = Nd4j.alloc;
if (allocationMode == AllocationMode.HEAP) {
if (copy) {
floatData = ArrayUtil.copy(data);
} else {
this.floatData = data;
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length);
wrappedBuffer.order(ByteOrder.nativeOrder());
FloatBuffer buffer = wrappedBuffer.asFloatBuffer();
for (int i = 0; i < data.length; i++) {
buffer.put(i, data[i]);
}
}
length = data.length;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(double[] data, boolean copy, int offset) {
this(data, copy);
this.offset = offset;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(double[] data, boolean copy) {
allocationMode = Nd4j.alloc;
if (allocationMode == AllocationMode.HEAP) {
if (copy) {
doubleData = ArrayUtil.copy(data);
} else {
this.doubleData = data;
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(8 * data.length);
wrappedBuffer.order(ByteOrder.nativeOrder());
DoubleBuffer buffer = wrappedBuffer.asDoubleBuffer();
for (int i = 0; i < data.length; i++) {
buffer.put(i, data[i]);
}
}
length = data.length;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(int[] data, boolean copy, int offset) {
this(data, copy);
this.offset = offset;
}
/**
* @param data
* @param copy
*/
public BaseDataBuffer(int[] data, boolean copy) {
allocationMode = Nd4j.alloc;
if (allocationMode == AllocationMode.HEAP) {
if (copy) intData = ArrayUtil.copy(data);
else this.intData = data;
} else {
wrappedBuffer = ByteBuffer.allocateDirect(4 * data.length);
wrappedBuffer.order(ByteOrder.nativeOrder());
IntBuffer buffer = wrappedBuffer.asIntBuffer();
for (int i = 0; i < data.length; i++) {
buffer.put(i, data[i]);
}
}
length = data.length;
}
/** @param data */
public BaseDataBuffer(double[] data) {
this(data, Nd4j.copyOnOps);
}
/** @param data */
public BaseDataBuffer(int[] data) {
this(data, Nd4j.copyOnOps);
}
/** @param data */
public BaseDataBuffer(float[] data) {
this(data, Nd4j.copyOnOps);
}
/**
* @param length
* @param elementSize
*/
public BaseDataBuffer(int length, int elementSize, int offset) {
this(length, elementSize);
this.offset = offset;
}
/**
* @param length
* @param elementSize
*/
public BaseDataBuffer(int length, int elementSize) {
allocationMode = Nd4j.alloc;
this.length = length;
this.elementSize = elementSize;
if (allocationMode() == AllocationMode.DIRECT) {
// allows for creation of the nio byte buffer to be overridden
setNioBuffer();
} else if (dataType() == Type.DOUBLE) {
doubleData = new double[length];
} else if (dataType() == Type.FLOAT) {
floatData = new float[length];
} else if (dataType() == Type.INT) intData = new int[length];
}
/**
* Create a data buffer from the given length
*
* @param buffer
* @param length
*/
public BaseDataBuffer(ByteBuffer buffer, int length, int offset) {
this(buffer, length);
this.offset = offset;
}
/**
* Create a data buffer from the given length
*
* @param buffer
* @param length
*/
public BaseDataBuffer(ByteBuffer buffer, int length) {
allocationMode = Nd4j.alloc;
this.length = length;
buffer.order(ByteOrder.nativeOrder());
if (allocationMode() == AllocationMode.DIRECT) {
this.wrappedBuffer = buffer;
} else if (dataType() == Type.INT) {
intData = new int[length];
IntBuffer intBuffer = buffer.asIntBuffer();
for (int i = 0; i < length; i++) {
intData[i] = intBuffer.get(i);
}
} else if (dataType() == Type.DOUBLE) {
doubleData = new double[length];
DoubleBuffer doubleBuffer = buffer.asDoubleBuffer();
for (int i = 0; i < length; i++) {
doubleData[i] = doubleBuffer.get(i);
}
} else if (dataType() == Type.FLOAT) {
floatData = new float[length];
FloatBuffer floatBuffer = buffer.asFloatBuffer();
for (int i = 0; i < length; i++) {
floatData[i] = floatBuffer.get(i);
}
}
}
// sets the nio wrapped buffer (allows to be overridden for other use cases like cuda)
protected void setNioBuffer() {
wrappedBuffer = ByteBuffer.allocateDirect(elementSize * length);
wrappedBuffer.order(ByteOrder.nativeOrder());
}
public BaseDataBuffer(byte[] data, int length) {
this(Unpooled.wrappedBuffer(data), length);
}
@Override
public int offset() {
return offset;
}
@Override
public AllocationMode allocationMode() {
return allocationMode;
}
@Override
public void persist() {
isPersist = true;
}
@Override
public boolean isPersist() {
return isPersist;
}
@Override
public void unPersist() {
isPersist = false;
}
/**
* Instantiate a buffer with the given length
*
* @param length the length of the buffer
*/
protected BaseDataBuffer(int length) {
this.length = length;
allocationMode = Nd4j.alloc;
if (length < 0) throw new IllegalArgumentException("Unable to create a buffer of length <= 0");
ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
if (allocationMode == AllocationMode.HEAP) {
if (length >= Integer.MAX_VALUE)
throw new IllegalArgumentException(
"Length of data buffer can not be > Integer.MAX_VALUE for heap (array based storage) allocation");
if (dataType() == Type.DOUBLE) doubleData = new double[length];
else if (dataType() == Type.FLOAT) floatData = new float[length];
} else {
if (length * getElementSize() < 0)
throw new IllegalArgumentException(
"Unable to create buffer of length " + length + " due to negative length specified");
wrappedBuffer =
ByteBuffer.allocateDirect(getElementSize() * length).order(ByteOrder.nativeOrder());
}
}
@Override
public void copyAtStride(
DataBuffer buf, int n, int stride, int yStride, int offset, int yOffset) {
if (dataType() == Type.FLOAT) {
for (int i = 0; i < n; i++) {
put(offset + i * stride, buf.getFloat(yOffset + i * yStride));
}
} else {
for (int i = 0; i < n; i++) {
put(offset + i * stride, buf.getDouble(yOffset + i * yStride));
}
}
}
@Override
public void removeReferencing(String id) {
referencing.remove(id);
}
@Override
public Collection<String> references() {
return referencing;
}
@Override
public void addReferencing(String id) {
referencing.add(id);
}
@Override
public void assign(int[] indices, float[] data, boolean contiguous, int inc) {
if (indices.length != data.length)
throw new IllegalArgumentException("Indices and data length must be the same");
if (indices.length > length())
throw new IllegalArgumentException(
"More elements than space to assign. This buffer is of length "
+ length()
+ " where the indices are of length "
+ data.length);
for (int i = 0; i < indices.length; i++) {
put(indices[i], data[i]);
}
}
@Override
public void setData(int[] data) {
if (intData != null) this.intData = data;
else {
for (int i = 0; i < data.length; i++) {
put(i, data[i]);
}
}
}
@Override
public void setData(float[] data) {
if (floatData != null) {
this.floatData = data;
} else {
for (int i = 0; i < data.length; i++) put(i, data[i]);
}
}
@Override
public void setData(double[] data) {
if (doubleData != null) {
this.doubleData = data;
} else {
for (int i = 0; i < data.length; i++) put(i, data[i]);
}
}
@Override
public void assign(int[] indices, double[] data, boolean contiguous, int inc) {
if (indices.length != data.length)
throw new IllegalArgumentException("Indices and data length must be the same");
if (indices.length > length())
throw new IllegalArgumentException(
"More elements than space to assign. This buffer is of length "
+ length()
+ " where the indices are of length "
+ data.length);
for (int i = 0; i < indices.length; i += inc) {
put(indices[i], data[i]);
}
}
@Override
public void assign(DataBuffer data) {
if (data.length() != length())
throw new IllegalArgumentException(
"Unable to assign buffer of length "
+ data.length()
+ " to this buffer of length "
+ length());
for (int i = 0; i < data.length(); i++) {
put(i, data.getDouble(i));
}
}
@Override
public void assign(int[] indices, float[] data, boolean contiguous) {
assign(indices, data, contiguous, 1);
}
@Override
public void assign(int[] indices, double[] data, boolean contiguous) {
assign(indices, data, contiguous, 1);
}
@Override
public int length() {
return length;
}
@Override
public void assign(Number value) {
for (int i = 0; i < length(); i++) assign(value, i);
}
@Override
public double[] getDoublesAt(int offset, int length) {
return getDoublesAt(offset, 1, length);
}
@Override
public float[] getFloatsAt(int offset, int inc, int length) {
if (offset + length > length()) length -= offset;
float[] ret = new float[length];
for (int i = 0; i < length; i++) {
ret[i] = getFloat(i + offset);
}
return ret;
}
@Override
public DataBuffer dup() {
if (floatData != null) {
return create(floatData);
} else if (doubleData != null) {
return create(doubleData);
} else if (intData != null) {
return create(intData);
}
DataBuffer ret = create(length);
for (int i = 0; i < ret.length(); i++) ret.put(i, getDouble(i));
return ret;
}
/**
* Create with length
*
* @param length a databuffer of the same type as this with the given length
* @return a data buffer with the same length and datatype as this one
*/
protected abstract DataBuffer create(int length);
/**
* Create the data buffer with respect to the given byte buffer
*
* @param data the buffer to create
* @return the data buffer based on the given buffer
*/
public abstract DataBuffer create(double[] data);
/**
* Create the data buffer with respect to the given byte buffer
*
* @param data the buffer to create
* @return the data buffer based on the given buffer
*/
public abstract DataBuffer create(float[] data);
/**
* Create the data buffer with respect to the given byte buffer
*
* @param data the buffer to create
* @return the data buffer based on the given buffer
*/
public abstract DataBuffer create(int[] data);
/**
* Create the data buffer with respect to the given byte buffer
*
* @param buf the buffer to create
* @return the data buffer based on the given buffer
*/
public abstract DataBuffer create(ByteBuf buf, int length);
@Override
public double[] getDoublesAt(int offset, int inc, int length) {
if (offset + length > length()) length -= offset;
double[] ret = new double[length];
for (int i = 0; i < length; i++) {
ret[i] = getDouble(i + offset);
}
return ret;
}
@Override
public float[] getFloatsAt(int offset, int length) {
return getFloatsAt(offset, 1, length);
}
@Override
public IComplexFloat getComplexFloat(int i) {
return Nd4j.createFloat(getFloat(i), getFloat(i + 1));
}
@Override
public IComplexDouble getComplexDouble(int i) {
return Nd4j.createDouble(getDouble(i), getDouble(i + 1));
}
@Override
public IComplexNumber getComplex(int i) {
return dataType() == DataBuffer.Type.FLOAT ? getComplexFloat(i) : getComplexDouble(i);
}
@Override
public void put(int i, IComplexNumber result) {
put(i, result.realComponent().doubleValue());
put(i + 1, result.imaginaryComponent().doubleValue());
}
@Override
public void assign(int[] offsets, int[] strides, DataBuffer... buffers) {
assign(offsets, strides, length(), buffers);
}
@Override
public byte[] asBytes() {
if (allocationMode == AllocationMode.HEAP) {
ByteArrayOutputStream bos = new ByteArrayOutputStream(getElementSize() * length());
DataOutputStream dos = new DataOutputStream(bos);
if (dataType() == Type.DOUBLE) {
if (doubleData == null) throw new IllegalStateException("Double array is null!");
try {
for (int i = 0; i < doubleData.length; i++) dos.writeDouble(doubleData[i]);
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
if (floatData == null) throw new IllegalStateException("Double array is null!");
try {
for (int i = 0; i < floatData.length; i++) dos.writeFloat(floatData[i]);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return bos.toByteArray();
} else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
if (dataType() == Type.DOUBLE) {
for (int i = 0; i < length(); i++) {
try {
dos.writeDouble(getDouble(i));
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
for (int i = 0; i < length(); i++) {
try {
dos.writeFloat(getFloat(i));
} catch (IOException e) {
e.printStackTrace();
}
}
}
return bos.toByteArray();
}
}
@Override
public float[] asFloat() {
if (allocationMode == AllocationMode.HEAP) {
if (floatData != null) {
return floatData;
}
}
float[] ret = new float[length];
for (int i = 0; i < length; i++) ret[i] = getFloat(i);
return ret;
}
@Override
public double[] asDouble() {
if (allocationMode == AllocationMode.HEAP) {
if (doubleData != null) {
return doubleData;
}
}
double[] ret = new double[length];
for (int i = 0; i < length; i++) ret[i] = getDouble(i);
return ret;
}
@Override
public int[] asInt() {
if (allocationMode == AllocationMode.HEAP) {
if (intData != null) {
return intData;
}
}
return wrappedBuffer.asIntBuffer().array();
}
@Override
public double getDouble(int i) {
if (doubleData != null) {
if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i);
dirty.set(false);
return doubleData[i];
} else if (floatData != null) {
if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i);
dirty.set(false);
return (double) floatData[i];
} else if (intData != null) {
dirty.set(false);
return (double) intData[i];
}
if (dataType() == Type.FLOAT) {
dirty.set(false);
return wrappedBuffer.asFloatBuffer().get(i);
} else if (dataType() == Type.INT) {
dirty.set(false);
return wrappedBuffer.asIntBuffer().get(i);
} else {
dirty.set(false);
return wrappedBuffer.asDoubleBuffer().get(i);
}
}
@Override
public float getFloat(int i) {
if (doubleData != null) {
if (i >= doubleData.length) throw new IllegalStateException("Index out of bounds " + i);
dirty.set(false);
return (float) doubleData[i];
} else if (floatData != null) {
if (i >= floatData.length) throw new IllegalStateException("Index out of bounds " + i);
dirty.set(false);
return floatData[i];
} else if (intData != null) {
dirty.set(false);
return (float) intData[i];
}
if (dataType() == Type.DOUBLE) {
dirty.set(false);
return (float) wrappedBuffer.asDoubleBuffer().get(i);
}
dirty.getAndSet(true);
return wrappedBuffer.asFloatBuffer().get(i);
}
@Override
public Number getNumber(int i) {
if (dataType() == Type.DOUBLE) return getDouble(i);
else if (dataType() == Type.INT) return getInt(i);
return getFloat(i);
}
@Override
public void put(int i, float element) {
put(i, (double) element);
}
@Override
public void put(int i, double element) {
if (i < 0 || i >= length()) throw new IllegalArgumentException("Illegal index " + i);
if (doubleData != null) doubleData[i] = element;
else if (floatData != null) floatData[i] = (float) element;
else if (intData != null) intData[i] = (int) element;
else {
if (dataType() == Type.DOUBLE) {
wrappedBuffer.asDoubleBuffer().put(i, element);
} else if (dataType() == Type.INT) {
wrappedBuffer.asIntBuffer().put(i, (int) element);
} else {
wrappedBuffer.asFloatBuffer().put(i, (float) element);
}
}
dirty.set(true);
}
@Override
public boolean dirty() {
return dirty.get();
}
@Override
public IntBuffer asNioInt() {
if (wrappedBuffer == null) {
return IntBuffer.wrap(intData);
}
return wrappedBuffer.asIntBuffer();
}
@Override
public DoubleBuffer asNioDouble() {
if (wrappedBuffer == null) {
return DoubleBuffer.wrap(doubleData);
}
return wrappedBuffer.asDoubleBuffer();
}
@Override
public FloatBuffer asNioFloat() {
if (wrappedBuffer == null) {
return FloatBuffer.wrap(floatData);
}
return wrappedBuffer.asFloatBuffer();
}
@Override
public ByteBuffer asNio() {
return wrappedBuffer;
}
@Override
public ByteBuf asNetty() {
if (wrappedBuffer != null) return Unpooled.wrappedBuffer(wrappedBuffer);
else if (floatData != null) return Unpooled.copyFloat(floatData);
else if (doubleData != null) return Unpooled.copyDouble(doubleData);
throw new IllegalStateException("No data source defined");
}
@Override
public void put(int i, int element) {
put(i, (double) element);
}
@Override
public void assign(Number value, int offset) {
for (int i = offset; i < length(); i++) put(i, value.doubleValue());
}
@Override
public void write(OutputStream dos) {
if (dos instanceof DataOutputStream) {
try {
write((DataOutputStream) dos);
} catch (IOException e) {
e.printStackTrace();
}
} else {
DataOutputStream dos2 = new DataOutputStream(dos);
try {
write(dos2);
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Override
public void read(InputStream is) {
if (is instanceof DataInputStream) {
read((DataInputStream) is);
} else {
DataInputStream dis2 = new DataInputStream(is);
read(dis2);
}
}
@Override
public void flush() {}
@Override
public int getInt(int ix) {
return (int) getDouble(ix);
}
@Override
public void assign(int[] offsets, int[] strides, long n, DataBuffer... buffers) {
if (offsets.length != strides.length || strides.length != buffers.length)
throw new IllegalArgumentException(
"Unable to assign buffers, please specify equal lengths strides, offsets, and buffers");
int length = 0;
for (int i = 0; i < buffers.length; i++) length += buffers[i].length();
int count = 0;
for (int i = 0; i < buffers.length; i++) {
for (int j = offsets[i]; j < buffers[i].length(); j += strides[i]) {
put(count++, buffers[i].getDouble(j));
}
}
if (count != n)
throw new IllegalArgumentException("Strides and offsets didn't match up to length " + n);
}
@Override
public void assign(DataBuffer... buffers) {
int[] offsets = new int[buffers.length];
int[] strides = new int[buffers.length];
for (int i = 0; i < strides.length; i++) strides[i] = 1;
assign(offsets, strides, buffers);
}
@Override
public void destroy() {}
@Override
public boolean equals(Object o) {
if (o instanceof DataBuffer) {
DataBuffer d = (DataBuffer) o;
if (d.length() != length()) return false;
for (int i = 0; i < length(); i++) {
double eps = Math.abs(getDouble(i) - d.getDouble(i));
if (eps > Nd4j.EPS_THRESHOLD) return false;
}
}
return true;
}
private void readObject(ObjectInputStream s) {
doReadObject(s);
}
private void writeObject(java.io.ObjectOutputStream out) throws IOException {
out.defaultWriteObject();
write(out);
}
protected void doReadObject(ObjectInputStream s) {
try {
s.defaultReadObject();
read(s);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected void read(DataInputStream s) {
try {
ref = new WeakReference<DataBuffer>(this, Nd4j.bufferRefQueue());
referencing = Collections.synchronizedSet(new HashSet<String>());
dirty = new AtomicBoolean(false);
allocationMode = AllocationMode.valueOf(s.readUTF());
length = s.readInt();
Type t = Type.valueOf(s.readUTF());
if (t == Type.DOUBLE) {
if (allocationMode == AllocationMode.HEAP) {
if (this.dataType() == Type.FLOAT) { // DataBuffer type
// double -> float
floatData = new float[length()];
} else if (this.dataType() == Type.DOUBLE) {
// double -> double
doubleData = new double[length()];
} else {
// double -> int
intData = new int[length()];
}
for (int i = 0; i < length(); i++) {
put(i, s.readDouble());
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
wrappedBuffer.order(ByteOrder.nativeOrder());
for (int i = 0; i < length(); i++) {
put(i, s.readDouble());
}
}
} else {
if (allocationMode == AllocationMode.HEAP) {
if (this.dataType() == Type.FLOAT) { // DataBuffer type
// float -> float
floatData = new float[length()];
} else if (this.dataType() == Type.DOUBLE) {
// float -> double
doubleData = new double[length()];
} else {
// float-> int
intData = new int[length()];
}
for (int i = 0; i < length(); i++) {
put(i, s.readFloat());
}
} else {
wrappedBuffer = ByteBuffer.allocateDirect(length() * getElementSize());
wrappedBuffer.order(ByteOrder.nativeOrder());
for (int i = 0; i < length(); i++) {
put(i, s.readFloat());
}
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected void write(DataOutputStream out) throws IOException {
out.writeUTF(allocationMode.name());
out.writeInt(length());
out.writeUTF(dataType().name());
if (dataType() == Type.DOUBLE) {
for (int i = 0; i < length(); i++) out.writeDouble(getDouble(i));
} else {
for (int i = 0; i < length(); i++) out.writeFloat(getFloat(i));
}
}
@Override
public Object array() {
if (floatData != null) return floatData;
if (doubleData != null) return doubleData;
throw new UnsupportedOperationException();
}
@Override
public String toString() {
StringBuffer ret = new StringBuffer();
ret.append("[");
for (int i = 0; i < length(); i++) {
ret.append(getNumber(i));
if (i < length() - 1) ret.append(",");
}
ret.append("]");
return ret.toString();
}
@Override
public int hashCode() {
int result = length;
result = 31 * result + (referencing != null ? referencing.hashCode() : 0);
result = 31 * result + (ref != null ? ref.hashCode() : 0);
result = 31 * result + (isPersist ? 1 : 0);
result = 31 * result + (allocationMode != null ? allocationMode.hashCode() : 0);
return result;
}
}
/** @return Map of versions to be verified. */
public Map<IgniteTxKey, GridCacheVersion> dhtVersions() {
return dhtVers == null ? Collections.<IgniteTxKey, GridCacheVersion>emptyMap() : dhtVers;
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}