/**
* Validate streams generate the same output.
*
* @param expIn Expected input stream.
* @param actIn Actual input stream.
* @param expSize Expected size of the streams.
* @param seek Seek to use async position-based reading or {@code null} to use simple continuous
* reading.
* @throws IOException In case of any IO exception.
*/
private void assertEqualStreams(
InputStream expIn, GridGgfsInputStream actIn, @Nullable Long expSize, @Nullable Long seek)
throws IOException {
if (seek != null) expIn.skip(seek);
int bufSize = 2345;
byte buf1[] = new byte[bufSize];
byte buf2[] = new byte[bufSize];
long pos = 0;
long start = System.currentTimeMillis();
while (true) {
int read = (int) Math.min(bufSize, expSize - pos);
int i1;
if (seek == null) i1 = actIn.read(buf1, 0, read);
else if (seek % 2 == 0) i1 = actIn.read(pos + seek, buf1, 0, read);
else {
i1 = read;
actIn.readFully(pos + seek, buf1, 0, read);
}
// Read at least 0 byte, but don't read more then 'i1' or 'read'.
int i2 = expIn.read(buf2, 0, Math.max(0, Math.min(i1, read)));
if (i1 != i2) {
fail(
"Expects the same data [read="
+ read
+ ", pos="
+ pos
+ ", seek="
+ seek
+ ", i1="
+ i1
+ ", i2="
+ i2
+ ']');
}
if (i1 == -1) break; // EOF
// i1 == bufSize => compare buffers.
// i1 < bufSize => Compare part of buffers, rest of buffers are equal from previous
// iteration.
assertTrue(
"Expects the same data [read="
+ read
+ ", pos="
+ pos
+ ", seek="
+ seek
+ ", i1="
+ i1
+ ", i2="
+ i2
+ ']',
Arrays.equals(buf1, buf2));
if (read == 0) break; // Nothing more to read.
pos += i1;
}
if (expSize != null) assertEquals(expSize.longValue(), pos);
long time = System.currentTimeMillis() - start;
if (time != 0 && log.isInfoEnabled()) {
log.info(
String.format(
"Streams were compared in continuous reading " + "[size=%7d, rate=%3.1f MB/sec]",
expSize, expSize * 1000. / time / 1024 / 1024));
}
}
/** @return Client configuration for the test. */
protected GridClientConfiguration clientConfiguration() throws GridClientException {
GridClientConfiguration cfg = new GridClientConfiguration();
cfg.setBalancer(getBalancer());
cfg.setTopologyRefreshFrequency(TOP_REFRESH_FREQ);
cfg.setProtocol(protocol());
cfg.setServers(Arrays.asList(serverAddress()));
cfg.setSslContextFactory(sslContextFactory());
GridClientDataConfiguration loc = new GridClientDataConfiguration();
GridClientDataConfiguration partitioned = new GridClientDataConfiguration();
partitioned.setName(PARTITIONED_CACHE_NAME);
partitioned.setAffinity(new GridClientPartitionAffinity());
GridClientDataConfiguration replicated = new GridClientDataConfiguration();
replicated.setName(REPLICATED_CACHE_NAME);
GridClientDataConfiguration replicatedAsync = new GridClientDataConfiguration();
replicatedAsync.setName(REPLICATED_ASYNC_CACHE_NAME);
cfg.setDataConfigurations(Arrays.asList(loc, partitioned, replicated, replicatedAsync));
return cfg;
}
/** {@inheritDoc} */
@Nullable
@Override
public <T> T value(CacheObjectContext ctx, boolean cpy) {
if (cpy) return (T) Arrays.copyOf(val, val.length);
return (T) val;
}
/** @throws Exception If failed. */
public void testClientAffinity() throws Exception {
GridClientData partitioned = client.data(PARTITIONED_CACHE_NAME);
Collection<Object> keys = new ArrayList<>();
keys.addAll(Arrays.asList(Boolean.TRUE, Boolean.FALSE, 1, Integer.MAX_VALUE));
Random rnd = new Random();
StringBuilder sb = new StringBuilder();
// Generate some random strings.
for (int i = 0; i < 100; i++) {
sb.setLength(0);
for (int j = 0; j < 255; j++)
// Only printable ASCII symbols for test.
sb.append((char) (rnd.nextInt(0x7f - 0x20) + 0x20));
keys.add(sb.toString());
}
// Generate some more keys to achieve better coverage.
for (int i = 0; i < 100; i++) keys.add(UUID.randomUUID());
for (Object key : keys) {
UUID nodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, key).id();
UUID clientNodeId = partitioned.affinity(key);
assertEquals(
"Invalid affinity mapping for REST response for key: " + key, nodeId, clientNodeId);
}
}
/** {@inheritDoc} */
@Override
public String toString() {
return S.toString(
GridNearAtomicUpdateRequest.class,
this,
"filter",
Arrays.toString(filter),
"parent",
super.toString());
}
/** {@inheritDoc} */
@Override
public Object onReceive(@Nullable Object obj) {
if (obj instanceof byte[]) {
X.println(">>> Byte array received over REST: " + Arrays.toString((byte[]) obj));
BigInteger val = new BigInteger((byte[]) obj);
X.println(">>> Unpacked a BigInteger from byte array received over REST: " + val);
return val;
} else return obj;
}
/** {@inheritDoc} */
@Override
public Object onSend(Object obj) {
if (obj instanceof BigInteger) {
X.println(">>> Creating byte array from BigInteger to send over REST: " + obj);
byte[] bytes = ((BigInteger) obj).toByteArray();
X.println(
">>> Created byte array from BigInteger to send over REST: " + Arrays.toString(bytes));
return bytes;
} else return obj;
}
/**
* Returns compact class host.
*
* @param obj Object to compact.
* @return String.
*/
@Nullable
public static Object compactObject(Object obj) {
if (obj == null) return null;
if (obj instanceof Enum) return obj.toString();
if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj;
if (obj instanceof Collection) {
Collection col = (Collection) obj;
Object[] res = new Object[col.size()];
int i = 0;
for (Object elm : col) res[i++] = compactObject(elm);
return res;
}
if (obj.getClass().isArray()) {
Class<?> arrType = obj.getClass().getComponentType();
if (arrType.isPrimitive()) {
if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj);
if (obj instanceof byte[]) return Arrays.toString((byte[]) obj);
if (obj instanceof short[]) return Arrays.toString((short[]) obj);
if (obj instanceof int[]) return Arrays.toString((int[]) obj);
if (obj instanceof long[]) return Arrays.toString((long[]) obj);
if (obj instanceof float[]) return Arrays.toString((float[]) obj);
if (obj instanceof double[]) return Arrays.toString((double[]) obj);
}
Object[] arr = (Object[]) obj;
int iMax = arr.length - 1;
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i <= iMax; i++) {
sb.append(compactObject(arr[i]));
if (i != iMax) sb.append(", ");
}
sb.append("]");
return sb.toString();
}
return U.compact(obj.getClass().getName());
}
/**
* Concat arrays in one.
*
* @param arrays Arrays.
* @return Summary array.
*/
public static int[] concat(int[]... arrays) {
assert arrays != null;
assert arrays.length > 1;
int len = 0;
for (int[] a : arrays) len += a.length;
int[] r = Arrays.copyOf(arrays[0], len);
for (int i = 1, shift = 0; i < arrays.length; i++) {
shift += arrays[i - 1].length;
System.arraycopy(arrays[i], 0, r, shift, arrays[i].length);
}
return r;
}
/**
* Run command in separated console.
*
* @param workFolder Work folder for command.
* @param args A string array containing the program and its arguments.
* @return Started process.
* @throws IOException If failed to start process.
*/
public static Process openInConsole(@Nullable File workFolder, String... args)
throws IOException {
String[] commands = args;
String cmd = F.concat(Arrays.asList(args), " ");
if (U.isWindows()) commands = F.asArray("cmd", "/c", String.format("start %s", cmd));
if (U.isMacOs())
commands =
F.asArray(
"osascript",
"-e",
String.format("tell application \"Terminal\" to do script \"%s\"", cmd));
if (U.isUnix()) commands = F.asArray("xterm", "-sl", "1024", "-geometry", "200x50", "-e", cmd);
ProcessBuilder pb = new ProcessBuilder(commands);
if (workFolder != null) pb.directory(workFolder);
return pb.start();
}
/**
* Constructs data loader peer-deploy aware.
*
* @param objs Collection of objects to detect deploy class and class loader.
*/
private DataLoaderPda(Object... objs) {
this.objs = Arrays.asList(objs);
}
/** @throws Exception If failed. */
public void testCreateFileFragmented() throws Exception {
GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs");
GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);
GridGgfsPath path = new GridGgfsPath("/file");
try {
GridGgfs fs0 = grid(0).ggfs("ggfs");
GridGgfs fs1 = grid(1).ggfs("ggfs");
GridGgfs fs2 = grid(2).ggfs("ggfs");
try (GridGgfsOutputStream out =
fs0.create(
path,
128,
false,
1,
CFG_GRP_SIZE,
F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 1);
out.write(data);
}
try (GridGgfsOutputStream out = fs1.append(path, false)) {
// 1.5 blocks.
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 2);
out.write(data);
}
// After this we should have first two block colocated with grid 0 and last block colocated
// with grid 1.
GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path);
GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME);
GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId());
GridGgfsFileMap map = fileInfo.fileMap();
List<GridGgfsFileAffinityRange> ranges = map.ranges();
assertEquals(2, ranges.size());
assertTrue(ranges.get(0).startOffset() == 0);
assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);
assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
// Validate data read after colocated writes.
try (GridGgfsInputStream in = fs2.open(path)) {
// Validate first part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());
// Validate second part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());
assertEquals(-1, in.read());
}
} finally {
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);
boolean hasData = false;
for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty();
assertTrue(hasData);
fs.delete(path, true);
}
GridTestUtils.retryAssert(
log,
ASSERT_RETRIES,
ASSERT_RETRY_INTERVAL,
new CAX() {
@Override
public void applyx() {
for (int i = 0; i < NODES_CNT; i++)
assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty());
}
});
}
/** @param lsnr Listener. */
void add(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) {
arr = Arrays.copyOf(arr, arr.length + 1);
arr[arr.length - 1] = lsnr;
}
/** {@inheritDoc} */
@Override
public Collection<UUID> masterNodeIds() {
return Arrays.asList(nodeId, nearNodeId);
}
/**
* Load client configuration from the properties map.
*
* @param prefix Prefix for the client properties.
* @param in Properties map to load configuration from.
* @throws GridClientException If parsing configuration failed.
*/
public void load(String prefix, Properties in) throws GridClientException {
while (prefix.endsWith(".")) prefix = prefix.substring(0, prefix.length() - 1);
if (!prefix.isEmpty()) prefix += ".";
String balancer = in.getProperty(prefix + "balancer");
String connectTimeout = in.getProperty(prefix + "connectTimeout");
String cred = in.getProperty(prefix + "credentials");
String autoFetchMetrics = in.getProperty(prefix + "autoFetchMetrics");
String autoFetchAttrs = in.getProperty(prefix + "autoFetchAttributes");
String maxConnIdleTime = in.getProperty(prefix + "idleTimeout");
String proto = in.getProperty(prefix + "protocol");
String srvrs = in.getProperty(prefix + "servers");
String tcpNoDelay = in.getProperty(prefix + "tcp.noDelay");
String topRefreshFreq = in.getProperty(prefix + "topology.refresh");
String sslEnabled = in.getProperty(prefix + "ssl.enabled");
String sslProto = in.getProperty(prefix + "ssl.protocol", "TLS");
String sslKeyAlg = in.getProperty(prefix + "ssl.key.algorithm", "SunX509");
String keyStorePath = in.getProperty(prefix + "ssl.keystore.location");
String keyStorePwd = in.getProperty(prefix + "ssl.keystore.password");
String keyStoreType = in.getProperty(prefix + "ssl.keystore.type");
String trustStorePath = in.getProperty(prefix + "ssl.truststore.location");
String trustStorePwd = in.getProperty(prefix + "ssl.truststore.password");
String trustStoreType = in.getProperty(prefix + "ssl.truststore.type");
String dataCfgs = in.getProperty(prefix + "data.configurations");
setBalancer(resolveBalancer(balancer));
if (!F.isEmpty(connectTimeout)) setConnectTimeout(Integer.parseInt(connectTimeout));
if (!F.isEmpty(cred)) {
int idx = cred.indexOf(':');
if (idx >= 0 && idx < cred.length() - 1) {
setSecurityCredentialsProvider(
new SecurityCredentialsBasicProvider(
new SecurityCredentials(cred.substring(0, idx), cred.substring(idx + 1))));
} else {
setSecurityCredentialsProvider(
new SecurityCredentialsBasicProvider(new SecurityCredentials(null, null, cred)));
}
}
if (!F.isEmpty(autoFetchMetrics)) setAutoFetchMetrics(Boolean.parseBoolean(autoFetchMetrics));
if (!F.isEmpty(autoFetchAttrs)) setAutoFetchAttributes(Boolean.parseBoolean(autoFetchAttrs));
if (!F.isEmpty(maxConnIdleTime)) setMaxConnectionIdleTime(Integer.parseInt(maxConnIdleTime));
if (!F.isEmpty(proto)) setProtocol(GridClientProtocol.valueOf(proto));
if (!F.isEmpty(srvrs)) setServers(Arrays.asList(srvrs.replaceAll("\\s+", "").split(",")));
if (!F.isEmpty(tcpNoDelay)) setTcpNoDelay(Boolean.parseBoolean(tcpNoDelay));
if (!F.isEmpty(topRefreshFreq)) setTopologyRefreshFrequency(Long.parseLong(topRefreshFreq));
//
// SSL configuration section
//
if (!F.isEmpty(sslEnabled) && Boolean.parseBoolean(sslEnabled)) {
GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
factory.setProtocol(F.isEmpty(sslProto) ? "TLS" : sslProto);
factory.setKeyAlgorithm(F.isEmpty(sslKeyAlg) ? "SunX509" : sslKeyAlg);
if (F.isEmpty(keyStorePath))
throw new IllegalArgumentException("SSL key store location is not specified.");
factory.setKeyStoreFilePath(keyStorePath);
if (keyStorePwd != null) factory.setKeyStorePassword(keyStorePwd.toCharArray());
factory.setKeyStoreType(F.isEmpty(keyStoreType) ? "jks" : keyStoreType);
if (F.isEmpty(trustStorePath))
factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
else {
factory.setTrustStoreFilePath(trustStorePath);
if (trustStorePwd != null) factory.setTrustStorePassword(trustStorePwd.toCharArray());
factory.setTrustStoreType(F.isEmpty(trustStoreType) ? "jks" : trustStoreType);
}
setSslContextFactory(factory);
}
//
// Data configuration section
//
if (!F.isEmpty(dataCfgs)) {
String[] names = dataCfgs.replaceAll("\\s+", "").split(",");
Collection<GridClientDataConfiguration> list = new ArrayList<>();
for (String cfgName : names) {
if (F.isEmpty(cfgName)) continue;
String name = in.getProperty(prefix + "data." + cfgName + ".name");
String bal = in.getProperty(prefix + "data." + cfgName + ".balancer");
String aff = in.getProperty(prefix + "data." + cfgName + ".affinity");
GridClientDataConfiguration dataCfg = new GridClientDataConfiguration();
dataCfg.setName(F.isEmpty(name) ? null : name);
dataCfg.setBalancer(resolveBalancer(bal));
dataCfg.setAffinity(resolveAffinity(aff));
list.add(dataCfg);
}
setDataConfigurations(list);
}
}
