public static void main(String[] args) throws IOException {
System.out.println("Input Stream:");
long start = System.currentTimeMillis();
Path filename = Paths.get(args[0]);
long crcValue = checksumInputStream(filename);
long end = System.currentTimeMillis();
System.out.println(Long.toHexString(crcValue));
System.out.println((end - start) + " milliseconds");
System.out.println("Buffered Input Stream:");
start = System.currentTimeMillis();
crcValue = checksumBufferedInputStream(filename);
end = System.currentTimeMillis();
System.out.println(Long.toHexString(crcValue));
System.out.println((end - start) + " milliseconds");
System.out.println("Random Access File:");
start = System.currentTimeMillis();
crcValue = checksumRandomAccessFile(filename);
end = System.currentTimeMillis();
System.out.println(Long.toHexString(crcValue));
System.out.println((end - start) + " milliseconds");
System.out.println("Mapped File:");
start = System.currentTimeMillis();
crcValue = checksumMappedFile(filename);
end = System.currentTimeMillis();
System.out.println(Long.toHexString(crcValue));
System.out.println((end - start) + " milliseconds");
}
static {
long time = System.currentTimeMillis();
ISubgraphTemplate graph = GoogleGraphLoader.constructGoogleGraph();
System.out.println("graph load: " + (System.currentTimeMillis() - time) / 1000.0);
time = System.currentTimeMillis();
_partition =
new BasePartition(
1,
graph,
PropertySet.EmptyPropertySet,
PropertySet.EmptyPropertySet,
Long2IntMaps.EMPTY_MAP,
new TestSubgraphFactory());
System.out.println("graph partitioning: " + (System.currentTimeMillis() - time) / 1000.0);
}
public void fileCreatedUpdated(boolean created, boolean modified) {
if (created) {
newFiles++;
}
if (modified) {
updatedFiles++;
}
recentModifications++;
processedFiles++;
// System.out.println("MediaIndexer.fileCreatedUpdated");
if (recentModifications > modificationsBeforeReport) {
System.out.println("\nProcessed files : " + processedFiles + "/" + totalNumberOfFiles);
System.out.println(
"Speed : "
+ (currentProgressSize / (System.currentTimeMillis() - lastProgressTime))
+ " MiB/s");
lastProgressTime = System.currentTimeMillis();
recentModifications = 0;
currentProgressSize = 0;
}
}
public void processMTRoot(String path) {
long t0 = System.currentTimeMillis();
DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
Date date = new Date();
try {
ts.addIndexPath(new File(path).getCanonicalPath());
System.out.println("MediaIndexer.processMTRoot()" + path);
System.out.println("MediaIndexer.processMTRoot() started at time " + dateFormat.format(date));
System.out.println("MediaIndexer.processMTRoot() computing number of files...");
totalNumberOfFiles = this.countFiles(path);
lastProgressTime = System.currentTimeMillis();
System.out.println("Number of files to explore " + totalNumberOfFiles);
if (executorService.isShutdown()) {
executorService =
new ThreadPoolExecutor(
maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, new LimitedQueue<Runnable>(50));
}
this.processedFiles = 0;
// this.processMT(new File(path));
TreeWalker t = new TreeWalker(this);
t.walk(path);
} catch (IOException e) {
e.printStackTrace();
}
executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
long t1 = System.currentTimeMillis();
date = new Date();
System.out.println("MediaIndexer.processMTRoot() finished at time " + dateFormat.format(date));
System.out.println("MediaIndexer.processMTRoot() found " + newFiles + " new files");
System.out.println("MediaIndexer.processMTRoot() updated " + updatedFiles + " files");
System.out.println("MediaIndexer.processMTRoot() total " + ts.size() + " files");
System.out.println("MediaIndexer.processMTRoot took " + (t1 - t0) / 1000 + " s");
}
public void doSerializationRoundtrip(String serializationType, SliceManager sliceManager)
throws IOException {
long time = System.currentTimeMillis();
long bytes = sliceManager.writePartition(_partition);
System.out.println(
serializationType
+ " serialization: "
+ (System.currentTimeMillis() - time) / 1000.0
+ "s , "
+ bytes / 1000.0
+ "kb");
time = System.currentTimeMillis();
IPartition actualPartition = sliceManager.readPartition();
System.out.println(
serializationType
+ " deserialization: "
+ (System.currentTimeMillis() - time) / 1000.0
+ "s");
assertEquals(_partition, actualPartition);
sliceManager.deletePartition();
}
// void processEvents() {
public void run() {
System.out.println("WatchDir Thread INFO: priority=" + Thread.currentThread().getPriority());
for (; ; ) {
// wait for key to be signalled
System.out.println("WatchDir INFO: restarting loop...acquiring new key");
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
return;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey not recognized!!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
System.out.println("Encountered OVERFLOW Event - " + event);
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
// print out event
System.out.format("[WatchDir] %s: %s\n", event.kind().name(), child);
// if directory is created, and watching recursively, then
// register it and its sub-directories
if (recursive && (kind == ENTRY_CREATE)) {
try {
if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
registerAll(child);
}
} catch (IOException x) {
// ignore to keep sample readbale
}
}
long t = System.currentTimeMillis();
if (!Folder.dontWatch.contains(Folder.getInternalPath(child))) {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
System.out.println(
"WatchDir#"
+ key
+ " INFO: path="
+ child
+ ", internal="
+ Folder.getInternalPath(child)
+ " is NOT in don't watch list. Forwarding it to other peers. @"
+ Main.timeToString(t)); // DEBUG
forwardToItopic(kind, child);
} else {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
System.out.println(
"WatchDir#"
+ key
+ " INFO: path="
+ child
+ ", internal="
+ Folder.getInternalPath(child)
+ " IS in the don't watch list. NOT forwarding. @"
+ Main.timeToString(t)); // DEBUG
// try{
// Thread.sleep(minDelayBtwnWatchEvents);
// } catch(InterruptedException ex) {
// System.err.println("Exception:"+ex+" while trying to sleep WatchDir thread");
// ex.printStackTrace();
// }
}
}
// reset key and remove from set if directory no longer accessible
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty()) {
break;
}
}
}
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}
/** Simple test to check exceptions and other cases */
@SuppressWarnings("unchecked")
static void testExceptions(Path dir) throws IOException {
System.out.println("-- Exceptions and other simple tests --");
WatchService watcher = FileSystems.getDefault().newWatchService();
try {
// Poll tests
WatchKey key;
System.out.println("poll...");
key = watcher.poll();
if (key != null) throw new RuntimeException("no keys registered");
System.out.println("poll with timeout...");
try {
long start = System.currentTimeMillis();
key = watcher.poll(3000, TimeUnit.MILLISECONDS);
if (key != null) throw new RuntimeException("no keys registered");
long waited = System.currentTimeMillis() - start;
if (waited < 2900) throw new RuntimeException("poll was too short");
} catch (InterruptedException x) {
throw new RuntimeException(x);
}
// IllegalArgumentException
System.out.println("IllegalArgumentException tests...");
try {
dir.register(watcher, new WatchEvent.Kind<?>[] {});
throw new RuntimeException("IllegalArgumentException not thrown");
} catch (IllegalArgumentException x) {
}
try {
// OVERFLOW is ignored so this is equivalent to the empty set
dir.register(watcher, new WatchEvent.Kind<?>[] {OVERFLOW});
throw new RuntimeException("IllegalArgumentException not thrown");
} catch (IllegalArgumentException x) {
}
// UnsupportedOperationException
try {
dir.register(
watcher,
new WatchEvent.Kind<?>[] {
new WatchEvent.Kind<Object>() {
@Override
public String name() {
return "custom";
}
@Override
public Class<Object> type() {
return Object.class;
}
}
});
} catch (UnsupportedOperationException x) {
}
try {
dir.register(
watcher,
new WatchEvent.Kind<?>[] {ENTRY_CREATE},
new WatchEvent.Modifier() {
@Override
public String name() {
return "custom";
}
});
throw new RuntimeException("UnsupportedOperationException not thrown");
} catch (UnsupportedOperationException x) {
}
// NullPointerException
System.out.println("NullPointerException tests...");
try {
dir.register(null, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
throw new RuntimeException("NullPointerException not thrown");
} catch (NullPointerException x) {
}
try {
dir.register(watcher, new WatchEvent.Kind<?>[] {null});
throw new RuntimeException("NullPointerException not thrown");
} catch (NullPointerException x) {
}
try {
dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE}, (WatchEvent.Modifier) null);
throw new RuntimeException("NullPointerException not thrown");
} catch (NullPointerException x) {
}
} finally {
watcher.close();
}
// -- ClosedWatchServiceException --
System.out.println("ClosedWatchServiceException tests...");
try {
watcher.poll();
throw new RuntimeException("ClosedWatchServiceException not thrown");
} catch (ClosedWatchServiceException x) {
}
// assume that poll throws exception immediately
long start = System.currentTimeMillis();
try {
watcher.poll(10000, TimeUnit.MILLISECONDS);
throw new RuntimeException("ClosedWatchServiceException not thrown");
} catch (InterruptedException x) {
throw new RuntimeException(x);
} catch (ClosedWatchServiceException x) {
long waited = System.currentTimeMillis() - start;
if (waited > 5000) throw new RuntimeException("poll was too long");
}
try {
watcher.take();
throw new RuntimeException("ClosedWatchServiceException not thrown");
} catch (InterruptedException x) {
throw new RuntimeException(x);
} catch (ClosedWatchServiceException x) {
}
try {
dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
throw new RuntimeException("ClosedWatchServiceException not thrown");
} catch (ClosedWatchServiceException x) {
}
System.out.println("OKAY");
}
