/**
* Joins array elements to string.
*
* @param arr Array.
* @return String.
*/
@Nullable
public static String compactArray(Object[] arr) {
if (arr == null || arr.length == 0) return null;
String sep = ", ";
StringBuilder sb = new StringBuilder();
for (Object s : arr) sb.append(s).append(sep);
if (sb.length() > 0) sb.setLength(sb.length() - sep.length());
return U.compact(sb.toString());
}
/**
* Log task mapped.
*
* @param log Logger.
* @param clazz Task class.
* @param nodes Mapped nodes.
*/
public static void logMapped(
@Nullable IgniteLogger log, Class<?> clazz, Collection<ClusterNode> nodes) {
log0(
log,
U.currentTimeMillis(),
String.format("[%s]: MAPPED: %s", clazz.getSimpleName(), U.toShortString(nodes)));
}
/**
* Decodes value from a given byte array to the object according to the flags given.
*
* @param flags Flags.
* @param bytes Byte array to decode.
* @return Decoded value.
* @throws GridException If deserialization failed.
*/
private Object decodeObj(short flags, byte[] bytes) throws GridException {
assert bytes != null;
if ((flags & SERIALIZED_FLAG) != 0)
return jdkMarshaller.unmarshal(new ByteArrayInputStream(bytes), null);
int masked = flags & 0xff00;
switch (masked) {
case BOOLEAN_FLAG:
return bytes[0] == '1';
case INT_FLAG:
return U.bytesToInt(bytes, 0);
case LONG_FLAG:
return U.bytesToLong(bytes, 0);
case DATE_FLAG:
return new Date(U.bytesToLong(bytes, 0));
case BYTE_FLAG:
return bytes[0];
case FLOAT_FLAG:
return Float.intBitsToFloat(U.bytesToInt(bytes, 0));
case DOUBLE_FLAG:
return Double.longBitsToDouble(U.bytesToLong(bytes, 0));
case BYTE_ARR_FLAG:
return bytes;
default:
return new String(bytes);
}
}
/**
* Encodes given object to a byte array and returns flags that describe the type of serialized
* object.
*
* @param obj Object to serialize.
* @param out Output stream to which object should be written.
* @return Serialization flags.
* @throws GridException If JDK serialization failed.
*/
private int encodeObj(Object obj, ByteArrayOutputStream out) throws GridException {
int flags = 0;
byte[] data = null;
if (obj instanceof String) {
data = ((String) obj).getBytes();
} else if (obj instanceof Boolean) {
data = new byte[] {(byte) ((Boolean) obj ? '1' : '0')};
flags |= BOOLEAN_FLAG;
} else if (obj instanceof Integer) {
data = U.intToBytes((Integer) obj);
flags |= INT_FLAG;
} else if (obj instanceof Long) {
data = U.longToBytes((Long) obj);
flags |= LONG_FLAG;
} else if (obj instanceof Date) {
data = U.longToBytes(((Date) obj).getTime());
flags |= DATE_FLAG;
} else if (obj instanceof Byte) {
data = new byte[] {(Byte) obj};
flags |= BYTE_FLAG;
} else if (obj instanceof Float) {
data = U.intToBytes(Float.floatToIntBits((Float) obj));
flags |= FLOAT_FLAG;
} else if (obj instanceof Double) {
data = U.longToBytes(Double.doubleToLongBits((Double) obj));
flags |= DOUBLE_FLAG;
} else if (obj instanceof byte[]) {
data = (byte[]) obj;
flags |= BYTE_ARR_FLAG;
} else {
jdkMarshaller.marshal(obj, out);
flags |= SERIALIZED_FLAG;
}
if (data != null) out.write(data, 0, data.length);
return flags;
}
/** {@inheritDoc} */
@Override
public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
throws IgniteCheckedException {
super.finishUnmarshal(ctx, ldr);
if (writes != null) unmarshalTx(writes, false, ctx, ldr);
if (reads != null) unmarshalTx(reads, false, ctx, ldr);
if (grpLockKeyBytes != null && grpLockKey == null)
grpLockKey = ctx.marshaller().unmarshal(grpLockKeyBytes, ldr);
if (dhtVerKeys != null && dhtVers == null) {
assert dhtVerVals != null;
assert dhtVerKeys.size() == dhtVerVals.size();
Iterator<IgniteTxKey> keyIt = dhtVerKeys.iterator();
Iterator<GridCacheVersion> verIt = dhtVerVals.iterator();
dhtVers = U.newHashMap(dhtVerKeys.size());
while (keyIt.hasNext()) {
IgniteTxKey key = keyIt.next();
key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
dhtVers.put(key, verIt.next());
}
}
if (txNodesBytes != null) txNodes = ctx.marshaller().unmarshal(txNodesBytes, ldr);
}
/**
* Resolves host for REST TCP server using grid configuration.
*
* @param cfg Grid configuration.
* @return REST host.
* @throws IOException If failed to resolve REST host.
*/
private InetAddress resolveRestTcpHost(GridConfiguration cfg) throws IOException {
String host = cfg.getRestTcpHost();
if (host == null) host = cfg.getLocalHost();
return U.resolveLocalHost(host);
}
/**
* Log finished.
*
* @param log Logger.
* @param clazz Class.
* @param start Start time.
*/
public static void logFinish(@Nullable IgniteLogger log, Class<?> clazz, long start) {
final long end = U.currentTimeMillis();
log0(
log,
end,
String.format(
"[%s]: FINISHED, duration: %s", clazz.getSimpleName(), formatDuration(end - start)));
}
/**
* Log message.
*
* @param log Logger.
* @param msg Message to log.
* @param clazz class.
* @param start start time.
* @return Time when message was logged.
*/
public static long log(@Nullable IgniteLogger log, String msg, Class<?> clazz, long start) {
final long end = U.currentTimeMillis();
log0(
log,
end,
String.format(
"[%s]: %s, duration: %s", clazz.getSimpleName(), msg, formatDuration(end - start)));
return end;
}
/**
* Returns compact class host.
*
* @param obj Object to compact.
* @return String.
*/
@Nullable
public static Object compactObject(Object obj) {
if (obj == null) return null;
if (obj instanceof Enum) return obj.toString();
if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj;
if (obj instanceof Collection) {
Collection col = (Collection) obj;
Object[] res = new Object[col.size()];
int i = 0;
for (Object elm : col) res[i++] = compactObject(elm);
return res;
}
if (obj.getClass().isArray()) {
Class<?> arrType = obj.getClass().getComponentType();
if (arrType.isPrimitive()) {
if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj);
if (obj instanceof byte[]) return Arrays.toString((byte[]) obj);
if (obj instanceof short[]) return Arrays.toString((short[]) obj);
if (obj instanceof int[]) return Arrays.toString((int[]) obj);
if (obj instanceof long[]) return Arrays.toString((long[]) obj);
if (obj instanceof float[]) return Arrays.toString((float[]) obj);
if (obj instanceof double[]) return Arrays.toString((double[]) obj);
}
Object[] arr = (Object[]) obj;
int iMax = arr.length - 1;
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i <= iMax; i++) {
sb.append(compactObject(arr[i]));
if (i != iMax) sb.append(", ");
}
sb.append("]");
return sb.toString();
}
return U.compact(obj.getClass().getName());
}
/**
* Run command in separated console.
*
* @param workFolder Work folder for command.
* @param args A string array containing the program and its arguments.
* @return Started process.
* @throws IOException If failed to start process.
*/
public static Process openInConsole(@Nullable File workFolder, String... args)
throws IOException {
String[] commands = args;
String cmd = F.concat(Arrays.asList(args), " ");
if (U.isWindows()) commands = F.asArray("cmd", "/c", String.format("start %s", cmd));
if (U.isMacOs())
commands =
F.asArray(
"osascript",
"-e",
String.format("tell application \"Terminal\" to do script \"%s\"", cmd));
if (U.isUnix()) commands = F.asArray("xterm", "-sl", "1024", "-geometry", "200x50", "-e", cmd);
ProcessBuilder pb = new ProcessBuilder(commands);
if (workFolder != null) pb.directory(workFolder);
return pb.start();
}
/**
* Resolve IGFS profiler logs directory.
*
* @param igfs IGFS instance to resolve logs dir for.
* @return {@link Path} to log dir or {@code null} if not found.
* @throws IgniteCheckedException if failed to resolve.
*/
public static Path resolveIgfsProfilerLogsDir(IgniteFileSystem igfs)
throws IgniteCheckedException {
String logsDir;
if (igfs instanceof IgfsEx) logsDir = ((IgfsEx) igfs).clientLogDirectory();
else if (igfs == null)
throw new IgniteCheckedException(
"Failed to get profiler log folder (IGFS instance not found)");
else
throw new IgniteCheckedException(
"Failed to get profiler log folder (unexpected IGFS instance type)");
URL logsDirUrl = U.resolveIgniteUrl(logsDir != null ? logsDir : DFLT_IGFS_LOG_DIR);
return logsDirUrl != null ? new File(logsDirUrl.getPath()).toPath() : null;
}
/** {@inheritDoc} */
@Override
public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
throws IgniteCheckedException {
super.finishUnmarshal(ctx, ldr);
if (ownedValKeys != null && ownedVals == null) {
ownedVals = U.newHashMap(ownedValKeys.size());
assert ownedValKeys.size() == ownedValVals.size();
Iterator<IgniteTxKey> keyIter = ownedValKeys.iterator();
Iterator<CacheVersionedValue> valIter = ownedValVals.iterator();
while (keyIter.hasNext()) {
IgniteTxKey key = keyIter.next();
GridCacheContext cctx = ctx.cacheContext(key.cacheId());
CacheVersionedValue val = valIter.next();
key.finishUnmarshal(cctx, ldr);
val.finishUnmarshal(cctx, ldr);
ownedVals.put(key, val);
}
}
if (retVal != null && retVal.cacheId() != 0) {
GridCacheContext cctx = ctx.cacheContext(retVal.cacheId());
assert cctx != null : retVal.cacheId();
retVal.finishUnmarshal(cctx, ldr);
}
if (filterFailedKeys != null) {
for (IgniteTxKey key : filterFailedKeys) {
GridCacheContext cctx = ctx.cacheContext(key.cacheId());
key.finishUnmarshal(cctx, ldr);
}
}
}
/**
* Read block from file.
*
* @param file - File to read.
* @param off - Marker position in file to start read from if {@code -1} read last blockSz bytes.
* @param blockSz - Maximum number of chars to read.
* @param lastModified - File last modification time.
* @return Read file block.
* @throws IOException In case of error.
*/
public static VisorFileBlock readBlock(File file, long off, int blockSz, long lastModified)
throws IOException {
RandomAccessFile raf = null;
try {
long fSz = file.length();
long fLastModified = file.lastModified();
long pos = off >= 0 ? off : Math.max(fSz - blockSz, 0);
// Try read more that file length.
if (fLastModified == lastModified && fSz != 0 && pos >= fSz)
throw new IOException(
"Trying to read file block with wrong offset: " + pos + " while file size: " + fSz);
if (fSz == 0)
return new VisorFileBlock(file.getPath(), pos, fLastModified, 0, false, EMPTY_FILE_BUF);
else {
int toRead = Math.min(blockSz, (int) (fSz - pos));
byte[] buf = new byte[toRead];
raf = new RandomAccessFile(file, "r");
raf.seek(pos);
int cntRead = raf.read(buf, 0, toRead);
if (cntRead != toRead)
throw new IOException(
"Count of requested and actually read bytes does not match [cntRead="
+ cntRead
+ ", toRead="
+ toRead
+ ']');
boolean zipped = buf.length > 512;
return new VisorFileBlock(
file.getPath(), pos, fSz, fLastModified, zipped, zipped ? zipBytes(buf) : buf);
}
} finally {
U.close(raf, null);
}
}
/**
* Checks if address can be reached using one argument InetAddress.isReachable() version or ping
* command if failed.
*
* @param addr Address to check.
* @param reachTimeout Timeout for the check.
* @return {@code True} if address is reachable.
*/
public static boolean reachableByPing(InetAddress addr, int reachTimeout) {
try {
if (addr.isReachable(reachTimeout)) return true;
String cmd = String.format("ping -%s 1 %s", U.isWindows() ? "n" : "c", addr.getHostAddress());
Process myProc = Runtime.getRuntime().exec(cmd);
myProc.waitFor();
return myProc.exitValue() == 0;
} catch (IOException ignore) {
return false;
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
return false;
}
}
/**
* Parses custom packet serialized by hessian marshaller.
*
* @param ses Session.
* @param buf Buffer containing not parsed bytes.
* @param state Parser state.
* @return Parsed message.
* @throws IOException If packet parsing or deserialization failed.
*/
@Nullable
private GridClientMessage parseCustomPacket(GridNioSession ses, ByteBuffer buf, ParserState state)
throws IOException {
assert state.packetType() == PacketType.GRIDGAIN;
assert state.packet() == null;
ByteArrayOutputStream tmp = state.buffer();
int len = state.index();
while (buf.remaining() > 0) {
byte b = buf.get();
if (len == 0) {
tmp.write(b);
if (tmp.size() == 4) {
len = U.bytesToInt(tmp.toByteArray(), 0);
tmp.reset();
if (len == 0) return PING_MESSAGE;
else if (len < 0)
throw new IOException(
"Failed to parse incoming packet (invalid packet length) [ses="
+ ses
+ ", len="
+ len
+ ']');
state.index(len);
}
} else {
tmp.write(b);
if (tmp.size() == len) return marshaller.unmarshal(tmp.toByteArray());
}
}
return null;
}
/**
* Decode file charset.
*
* @param f File to process.
* @return File charset.
* @throws IOException in case of error.
*/
public static Charset decode(File f) throws IOException {
SortedMap<String, Charset> charsets = Charset.availableCharsets();
String[] firstCharsets = {
Charset.defaultCharset().name(), "US-ASCII", "UTF-8", "UTF-16BE", "UTF-16LE"
};
Collection<Charset> orderedCharsets = U.newLinkedHashSet(charsets.size());
for (String c : firstCharsets)
if (charsets.containsKey(c)) orderedCharsets.add(charsets.get(c));
orderedCharsets.addAll(charsets.values());
try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
FileChannel ch = raf.getChannel();
ByteBuffer buf = ByteBuffer.allocate(4096);
ch.read(buf);
buf.flip();
for (Charset charset : orderedCharsets) {
CharsetDecoder decoder = charset.newDecoder();
decoder.reset();
try {
decoder.decode(buf);
return charset;
} catch (CharacterCodingException ignored) {
}
}
}
return Charset.defaultCharset();
}
/** {@inheritDoc} */
@Override
public ByteBuffer encode(GridNioSession ses, GridClientMessage msg)
throws IOException, GridException {
assert msg != null;
if (msg instanceof GridTcpRestPacket) return encodeMemcache((GridTcpRestPacket) msg);
else if (msg == PING_MESSAGE) return ByteBuffer.wrap(PING_PACKET);
else {
byte[] data = marshaller.marshal(msg);
assert data.length > 0;
ByteBuffer res = ByteBuffer.allocate(data.length + 5);
res.put(GRIDGAIN_REQ_FLAG);
res.put(U.intToBytes(data.length));
res.put(data);
res.flip();
return res;
}
}
/** {@inheritDoc} */
@Override
public void writeExternal(ObjectOutput out) throws IOException {
U.writeByteArray(out, val);
}
/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
val = U.readByteArray(in);
}
/**
* Validates incoming packet and deserializes all fields that need to be deserialized.
*
* @param ses Session on which packet is being parsed.
* @param req Raw packet.
* @return Same packet with fields deserialized.
* @throws IOException If parsing failed.
* @throws GridException If deserialization failed.
*/
private GridClientMessage assemble(GridNioSession ses, GridTcpRestPacket req)
throws IOException, GridException {
byte[] extras = req.extras();
// First, decode key and value, if any
if (req.key() != null || req.value() != null) {
short keyFlags = 0;
short valFlags = 0;
if (req.hasFlags()) {
if (extras == null || extras.length < FLAGS_LENGTH)
throw new IOException(
"Failed to parse incoming packet (flags required for command) [ses="
+ ses
+ ", opCode="
+ Integer.toHexString(req.operationCode() & 0xFF)
+ ']');
keyFlags = U.bytesToShort(extras, 0);
valFlags = U.bytesToShort(extras, 2);
}
if (req.key() != null) {
assert req.key() instanceof byte[];
byte[] rawKey = (byte[]) req.key();
// Only values can be hessian-encoded.
req.key(decodeObj(keyFlags, rawKey));
}
if (req.value() != null) {
assert req.value() instanceof byte[];
byte[] rawVal = (byte[]) req.value();
req.value(decodeObj(valFlags, rawVal));
}
}
if (req.hasExpiration()) {
if (extras == null || extras.length < 8)
throw new IOException(
"Failed to parse incoming packet (expiration value required for command) [ses="
+ ses
+ ", opCode="
+ Integer.toHexString(req.operationCode() & 0xFF)
+ ']');
req.expiration(U.bytesToInt(extras, 4) & 0xFFFFFFFFL);
}
if (req.hasInitial()) {
if (extras == null || extras.length < 16)
throw new IOException(
"Failed to parse incoming packet (initial value required for command) [ses="
+ ses
+ ", opCode="
+ Integer.toHexString(req.operationCode() & 0xFF)
+ ']');
req.initial(U.bytesToLong(extras, 8));
}
if (req.hasDelta()) {
if (extras == null || extras.length < 8)
throw new IOException(
"Failed to parse incoming packet (delta value required for command) [ses="
+ ses
+ ", opCode="
+ Integer.toHexString(req.operationCode() & 0xFF)
+ ']');
req.delta(U.bytesToLong(extras, 0));
}
if (extras != null) {
// Clients that include cache name must always include flags.
int length = 4;
if (req.hasExpiration()) length += 4;
if (req.hasDelta()) length += 8;
if (req.hasInitial()) length += 8;
if (extras.length - length > 0) {
byte[] cacheName = new byte[extras.length - length];
System.arraycopy(extras, length, cacheName, 0, extras.length - length);
req.cacheName(new String(cacheName));
}
}
return req;
}
/**
* Parses memcache protocol message.
*
* @param ses Session.
* @param buf Buffer containing not parsed bytes.
* @param state Current parser state.
* @return Parsed packet.s
* @throws IOException If packet cannot be parsed.
* @throws GridException If deserialization error occurred.
*/
@Nullable
private GridClientMessage parseMemcachePacket(
GridNioSession ses, ByteBuffer buf, ParserState state) throws IOException, GridException {
assert state.packetType() == PacketType.MEMCACHE;
assert state.packet() != null;
assert state.packet() instanceof GridTcpRestPacket;
GridTcpRestPacket req = (GridTcpRestPacket) state.packet();
ByteArrayOutputStream tmp = state.buffer();
int i = state.index();
while (buf.remaining() > 0) {
byte b = buf.get();
if (i == 0) req.requestFlag(b);
else if (i == 1) req.operationCode(b);
else if (i == 2 || i == 3) {
tmp.write(b);
if (i == 3) {
req.keyLength(U.bytesToShort(tmp.toByteArray(), 0));
tmp.reset();
}
} else if (i == 4) req.extrasLength(b);
else if (i >= 8 && i <= 11) {
tmp.write(b);
if (i == 11) {
req.totalLength(U.bytesToInt(tmp.toByteArray(), 0));
tmp.reset();
}
} else if (i >= 12 && i <= 15) {
tmp.write(b);
if (i == 15) {
req.opaque(tmp.toByteArray());
tmp.reset();
}
} else if (i >= HDR_LEN && i < HDR_LEN + req.extrasLength()) {
tmp.write(b);
if (i == HDR_LEN + req.extrasLength() - 1) {
req.extras(tmp.toByteArray());
tmp.reset();
}
} else if (i >= HDR_LEN + req.extrasLength()
&& i < HDR_LEN + req.extrasLength() + req.keyLength()) {
tmp.write(b);
if (i == HDR_LEN + req.extrasLength() + req.keyLength() - 1) {
req.key(tmp.toByteArray());
tmp.reset();
}
} else if (i >= HDR_LEN + req.extrasLength() + req.keyLength()
&& i < HDR_LEN + req.totalLength()) {
tmp.write(b);
if (i == HDR_LEN + req.totalLength() - 1) {
req.value(tmp.toByteArray());
tmp.reset();
}
}
if (i == HDR_LEN + req.totalLength() - 1)
// Assembled the packet.
return assemble(ses, req);
i++;
}
state.index(i);
return null;
}
/** {@inheritDoc} */
@SuppressWarnings("BusyWait")
@Override
public void start(final GridRestProtocolHandler hnd) throws GridException {
assert hnd != null;
GridConfiguration cfg = ctx.config();
GridNioServerListener<GridClientMessage> lsnr = new GridTcpRestNioListener(log, hnd);
GridNioParser parser = new GridTcpRestParser(log);
try {
host = resolveRestTcpHost(cfg);
SSLContext sslCtx = null;
if (cfg.isRestTcpSslEnabled()) {
GridSslContextFactory factory = cfg.getRestTcpSslContextFactory();
if (factory == null)
// Thrown SSL exception instead of GridException for writing correct warning message into
// log.
throw new SSLException("SSL is enabled, but SSL context factory is not specified.");
sslCtx = factory.createSslContext();
}
int lastPort = cfg.getRestTcpPort() + cfg.getRestPortRange() - 1;
for (port = cfg.getRestTcpPort(); port <= lastPort; port++) {
if (startTcpServer(host, port, lsnr, parser, sslCtx, cfg)) {
if (log.isInfoEnabled()) log.info(startInfo());
return;
}
}
U.warn(
log,
"Failed to start TCP binary REST server (possibly all ports in range are in use) "
+ "[firstPort="
+ cfg.getRestTcpPort()
+ ", lastPort="
+ lastPort
+ ", host="
+ host
+ ']');
} catch (SSLException e) {
U.warn(
log,
"Failed to start " + name() + " protocol on port " + port + ": " + e.getMessage(),
"Failed to start "
+ name()
+ " protocol on port "
+ port
+ ". Check if SSL context factory is "
+ "properly configured.");
} catch (IOException e) {
U.warn(
log,
"Failed to start " + name() + " protocol on port " + port + ": " + e.getMessage(),
"Failed to start "
+ name()
+ " protocol on port "
+ port
+ ". "
+ "Check restTcpHost configuration property.");
}
}
/**
* Compact class names.
*
* @param obj Object for compact.
* @return Compacted string.
*/
@Nullable
public static String compactClass(@Nullable Object obj) {
if (obj == null) return null;
return U.compact(obj.getClass().getName());
}
/**
* Log message.
*
* @param log Logger.
* @param msg Message.
*/
public static void log(@Nullable IgniteLogger log, String msg) {
log0(log, U.currentTimeMillis(), " " + msg);
}