/**
* Checks if address can be reached using one argument InetAddress.isReachable() version or ping
* command if failed.
*
* @param addr Address to check.
* @param reachTimeout Timeout for the check.
* @return {@code True} if address is reachable.
*/
public static boolean reachableByPing(InetAddress addr, int reachTimeout) {
try {
if (addr.isReachable(reachTimeout)) return true;
String cmd = String.format("ping -%s 1 %s", U.isWindows() ? "n" : "c", addr.getHostAddress());
Process myProc = Runtime.getRuntime().exec(cmd);
myProc.waitFor();
return myProc.exitValue() == 0;
} catch (IOException ignore) {
return false;
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
return false;
}
}
/**
* This adepter designed to support stores with bulk loading from stream-like source.
*
* <p>This class processes input data in the following way:
*
* <ul>
* <li>Iterator of input record obtained from user-defined {@link #inputIterator(Object...)}.
* <li>Iterator continuously queried for input records and they are grouped into batches of {@link
* #batchSize}.
* <li>Batch is placed into processing queue and puled by one of {@link #threadsCnt} working
* threads.
* <li>Each record in batch is passed to user-defined {@link #parse(Object, Object...)} method and
* result is stored into cache.
* </ul>
*
* <p>Two methods should be implemented by inheritants:
*
* <ul>
* <li>{@link #inputIterator(Object...)}. It should open underlying data source and iterate all
* record available in it. Individual records could be in very raw form, like text lines for
* CSV files.
* <li>{@link #parse(Object, Object...)}. This method should process input records and transform
* them into key-value pairs for cache.
* </ul>
*
* <p>
*
* @param <K> Key type.
* @param <V> Value type.
* @param <I> Input type.
* @author @java.author
* @version @java.version
*/
public abstract class GridCacheLoadOnlyStoreAdapter<K, V, I> implements GridCacheStore<K, V> {
/**
* Default batch size (number of records read with {@link #inputIterator(Object...)} and then
* submitted to internal pool at a time).
*/
public static final int DFLT_BATCH_SIZE = 100;
/** Default batch queue size (max batches count to limit memory usage). */
public static final int DFLT_BATCH_QUEUE_SIZE = 100;
/** Default number of working threads (equal to the number of available processors). */
public static final int DFLT_THREADS_COUNT = Runtime.getRuntime().availableProcessors();
/** Auto-injected logger. */
@GridLoggerResource private GridLogger log;
/** Batch size. */
private int batchSize = DFLT_BATCH_SIZE;
/** Size of queue of batches to process. */
private int batchQueueSize = DFLT_BATCH_QUEUE_SIZE;
/** Number fo working threads. */
private int threadsCnt = DFLT_THREADS_COUNT;
/**
* Returns iterator of input records.
*
* <p>Note that returned iterator doesn't have to be thread-safe. Thus it could operate on raw
* streams, DB connections, etc. without additional synchronization.
*
* @param args Arguments passes into {@link GridCache#loadCache(GridBiPredicate, long, Object...)}
* method.
* @return Iterator over input records.
* @throws GridException If iterator can't be created with the given arguments.
*/
protected abstract Iterator<I> inputIterator(@Nullable Object... args) throws GridException;
/**
* This method should transform raw data records into valid key-value pairs to be stored into
* cache.
*
* <p>If {@code null} is returned then this record will be just skipped.
*
* @param rec A raw data record.
* @param args Arguments passed into {@link GridCache#loadCache(GridBiPredicate, long, Object...)}
* method.
* @return Cache entry to be saved in cache or {@code null} if no entry could be produced from
* this record.
*/
@Nullable
protected abstract GridBiTuple<K, V> parse(I rec, @Nullable Object... args);
/** {@inheritDoc} */
@Override
public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
ExecutorService exec =
new ThreadPoolExecutor(
threadsCnt,
threadsCnt,
0L,
MILLISECONDS,
new ArrayBlockingQueue<Runnable>(batchQueueSize),
new BlockingRejectedExecutionHandler());
Iterator<I> iter = inputIterator(args);
Collection<I> buf = new ArrayList<>(batchSize);
try {
while (iter.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
U.warn(log, "Working thread was interrupted while loading data.");
break;
}
buf.add(iter.next());
if (buf.size() == batchSize) {
exec.submit(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
} catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
assert false : "RejectedExecutionException was thrown while it shouldn't.";
} finally {
exec.shutdown();
try {
exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");
Thread.currentThread().interrupt();
}
}
}
/**
* Returns batch size.
*
* @return Batch size.
*/
public int getBatchSize() {
return batchSize;
}
/**
* Sets batch size.
*
* @param batchSize Batch size.
*/
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
/**
* Returns batch queue size.
*
* @return Batch queue size.
*/
public int getBatchQueueSize() {
return batchQueueSize;
}
/**
* Sets batch queue size.
*
* @param batchQueueSize Batch queue size.
*/
public void setBatchQueueSize(int batchQueueSize) {
this.batchQueueSize = batchQueueSize;
}
/**
* Returns number of worker threads.
*
* @return Number of worker threads.
*/
public int getThreadsCount() {
return threadsCnt;
}
/**
* Sets number of worker threads.
*
* @param threadsCnt Number of worker threads.
*/
public void setThreadsCount(int threadsCnt) {
this.threadsCnt = threadsCnt;
}
/** {@inheritDoc} */
@Override
public V load(@Nullable GridCacheTx tx, K key) throws GridException {
return null;
}
/** {@inheritDoc} */
@Override
public void loadAll(
@Nullable GridCacheTx tx, @Nullable Collection<? extends K> keys, GridBiInClosure<K, V> c)
throws GridException {
// No-op.
}
/** {@inheritDoc} */
@Override
public void put(@Nullable GridCacheTx tx, K key, @Nullable V val) throws GridException {
// No-op.
}
/** {@inheritDoc} */
@Override
public void putAll(@Nullable GridCacheTx tx, @Nullable Map<? extends K, ? extends V> map)
throws GridException {
// No-op.
}
/** {@inheritDoc} */
@Override
public void remove(@Nullable GridCacheTx tx, K key) throws GridException {
// No-op.
}
/** {@inheritDoc} */
@Override
public void removeAll(@Nullable GridCacheTx tx, @Nullable Collection<? extends K> keys)
throws GridException {
// No-op.
}
/** {@inheritDoc} */
@Override
public void txEnd(GridCacheTx tx, boolean commit) throws GridException {
// No-op.
}
/** Worker. */
private class Worker implements Runnable {
/** */
private final GridBiInClosure<K, V> c;
/** */
private final Collection<I> buf;
/** */
private final Object[] args;
/**
* @param c Closure for loaded entries.
* @param buf Set of input records to process.
* @param args Arguments passed into {@link GridCache#loadCache(GridBiPredicate, long,
* Object...)} method.
*/
Worker(GridBiInClosure<K, V> c, Collection<I> buf, Object[] args) {
this.c = c;
this.buf = buf;
this.args = args;
}
/** {@inheritDoc} */
@Override
public void run() {
for (I rec : buf) {
GridBiTuple<K, V> entry = parse(rec, args);
if (entry != null) c.apply(entry.getKey(), entry.getValue());
}
}
}
/**
* This handler blocks the caller thread until free space will be available in tasks queue. If the
* executor is shut down than it throws {@link RejectedExecutionException}.
*
* <p>It is save to apply this policy when:
*
* <ol>
* <li>{@code shutdownNow} is not used on the pool.
* <li>{@code shutdown} is called from the thread where all submissions where performed.
* </ol>
*/
private class BlockingRejectedExecutionHandler implements RejectedExecutionHandler {
/** {@inheritDoc} */
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
if (executor.isShutdown()) throw new RejectedExecutionException();
else executor.getQueue().put(r);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while loading data.");
Thread.currentThread().interrupt();
}
}
}
}
