Пример #1
0
  public static void main(String[] args) throws IOException {
    System.out.println("Input Stream:");
    long start = System.currentTimeMillis();
    Path filename = Paths.get(args[0]);
    long crcValue = checksumInputStream(filename);
    long end = System.currentTimeMillis();
    System.out.println(Long.toHexString(crcValue));
    System.out.println((end - start) + " milliseconds");

    System.out.println("Buffered Input Stream:");
    start = System.currentTimeMillis();
    crcValue = checksumBufferedInputStream(filename);
    end = System.currentTimeMillis();
    System.out.println(Long.toHexString(crcValue));
    System.out.println((end - start) + " milliseconds");

    System.out.println("Random Access File:");
    start = System.currentTimeMillis();
    crcValue = checksumRandomAccessFile(filename);
    end = System.currentTimeMillis();
    System.out.println(Long.toHexString(crcValue));
    System.out.println((end - start) + " milliseconds");

    System.out.println("Mapped File:");
    start = System.currentTimeMillis();
    crcValue = checksumMappedFile(filename);
    end = System.currentTimeMillis();
    System.out.println(Long.toHexString(crcValue));
    System.out.println((end - start) + " milliseconds");
  }
  static {
    long time = System.currentTimeMillis();
    ISubgraphTemplate graph = GoogleGraphLoader.constructGoogleGraph();
    System.out.println("graph load: " + (System.currentTimeMillis() - time) / 1000.0);

    time = System.currentTimeMillis();
    _partition =
        new BasePartition(
            1,
            graph,
            PropertySet.EmptyPropertySet,
            PropertySet.EmptyPropertySet,
            Long2IntMaps.EMPTY_MAP,
            new TestSubgraphFactory());
    System.out.println("graph partitioning: " + (System.currentTimeMillis() - time) / 1000.0);
  }
Пример #3
0
 public void fileCreatedUpdated(boolean created, boolean modified) {
   if (created) {
     newFiles++;
   }
   if (modified) {
     updatedFiles++;
   }
   recentModifications++;
   processedFiles++;
   //  System.out.println("MediaIndexer.fileCreatedUpdated");
   if (recentModifications > modificationsBeforeReport) {
     System.out.println("\nProcessed files : " + processedFiles + "/" + totalNumberOfFiles);
     System.out.println(
         "Speed : "
             + (currentProgressSize / (System.currentTimeMillis() - lastProgressTime))
             + " MiB/s");
     lastProgressTime = System.currentTimeMillis();
     recentModifications = 0;
     currentProgressSize = 0;
   }
 }
Пример #4
0
  public void processMTRoot(String path) {
    long t0 = System.currentTimeMillis();
    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    Date date = new Date();
    try {
      ts.addIndexPath(new File(path).getCanonicalPath());
      System.out.println("MediaIndexer.processMTRoot()" + path);
      System.out.println("MediaIndexer.processMTRoot() started at time " + dateFormat.format(date));
      System.out.println("MediaIndexer.processMTRoot() computing number of files...");
      totalNumberOfFiles = this.countFiles(path);
      lastProgressTime = System.currentTimeMillis();
      System.out.println("Number of files to explore " + totalNumberOfFiles);
      if (executorService.isShutdown()) {
        executorService =
            new ThreadPoolExecutor(
                maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, new LimitedQueue<Runnable>(50));
      }

      this.processedFiles = 0;
      // this.processMT(new File(path));
      TreeWalker t = new TreeWalker(this);
      t.walk(path);
    } catch (IOException e) {
      e.printStackTrace();
    }
    executorService.shutdown();
    try {
      executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
    long t1 = System.currentTimeMillis();
    date = new Date();
    System.out.println("MediaIndexer.processMTRoot() finished at time " + dateFormat.format(date));
    System.out.println("MediaIndexer.processMTRoot() found " + newFiles + " new files");
    System.out.println("MediaIndexer.processMTRoot() updated " + updatedFiles + " files");
    System.out.println("MediaIndexer.processMTRoot() total " + ts.size() + " files");
    System.out.println("MediaIndexer.processMTRoot took " + (t1 - t0) / 1000 + " s");
  }
  public void doSerializationRoundtrip(String serializationType, SliceManager sliceManager)
      throws IOException {
    long time = System.currentTimeMillis();
    long bytes = sliceManager.writePartition(_partition);
    System.out.println(
        serializationType
            + " serialization: "
            + (System.currentTimeMillis() - time) / 1000.0
            + "s , "
            + bytes / 1000.0
            + "kb");

    time = System.currentTimeMillis();
    IPartition actualPartition = sliceManager.readPartition();
    System.out.println(
        serializationType
            + " deserialization: "
            + (System.currentTimeMillis() - time) / 1000.0
            + "s");

    assertEquals(_partition, actualPartition);

    sliceManager.deletePartition();
  }
Пример #6
0
  // void processEvents() {
  public void run() {
    System.out.println("WatchDir Thread INFO: priority=" + Thread.currentThread().getPriority());
    for (; ; ) {
      // wait for key to be signalled
      System.out.println("WatchDir INFO: restarting loop...acquiring new key");
      WatchKey key;
      try {
        key = watcher.take();
      } catch (InterruptedException x) {
        return;
      }

      Path dir = keys.get(key);
      if (dir == null) {
        System.err.println("WatchKey not recognized!!");
        continue;
      }

      for (WatchEvent<?> event : key.pollEvents()) {
        WatchEvent.Kind kind = event.kind();
        // TBD - provide example of how OVERFLOW event is handled
        if (kind == OVERFLOW) {
          System.out.println("Encountered OVERFLOW Event - " + event);
          continue;
        }

        // Context for directory entry event is the file name of entry
        WatchEvent<Path> ev = cast(event);
        Path name = ev.context();
        Path child = dir.resolve(name);

        // print out event
        System.out.format("[WatchDir] %s: %s\n", event.kind().name(), child);

        // if directory is created, and watching recursively, then
        // register it and its sub-directories
        if (recursive && (kind == ENTRY_CREATE)) {
          try {
            if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
              registerAll(child);
            }
          } catch (IOException x) {
            // ignore to keep sample readbale
          }
        }

        long t = System.currentTimeMillis();
        if (!Folder.dontWatch.contains(Folder.getInternalPath(child))) {
          Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
          System.out.println(
              "WatchDir#"
                  + key
                  + " INFO: path="
                  + child
                  + ", internal="
                  + Folder.getInternalPath(child)
                  + " is NOT in don't watch list. Forwarding it to other peers. @"
                  + Main.timeToString(t)); // DEBUG
          forwardToItopic(kind, child);
        } else {
          Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
          System.out.println(
              "WatchDir#"
                  + key
                  + " INFO: path="
                  + child
                  + ", internal="
                  + Folder.getInternalPath(child)
                  + " IS in the don't watch list. NOT forwarding. @"
                  + Main.timeToString(t)); // DEBUG
          // try{
          //     Thread.sleep(minDelayBtwnWatchEvents);
          // } catch(InterruptedException ex) {
          //     System.err.println("Exception:"+ex+" while trying to sleep WatchDir thread");
          //     ex.printStackTrace();
          // }
        }
      }

      // reset key and remove from set if directory no longer accessible
      boolean valid = key.reset();
      if (!valid) {
        keys.remove(key);

        // all directories are inaccessible
        if (keys.isEmpty()) {
          break;
        }
      }
    }
  }
Пример #7
0
  /**
   * Grabs local events and detects if events was lost since last poll.
   *
   * @param ignite Target grid.
   * @param evtOrderKey Unique key to take last order key from node local map.
   * @param evtThrottleCntrKey Unique key to take throttle count from node local map.
   * @param evtTypes Event types to collect.
   * @param evtMapper Closure to map grid events to Visor data transfer objects.
   * @return Collections of node events
   */
  public static Collection<VisorGridEvent> collectEvents(
      Ignite ignite,
      String evtOrderKey,
      String evtThrottleCntrKey,
      final int[] evtTypes,
      IgniteClosure<Event, VisorGridEvent> evtMapper) {
    assert ignite != null;
    assert evtTypes != null && evtTypes.length > 0;

    ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();

    final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
    final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);

    // When we first time arrive onto a node to get its local events,
    // we'll grab only last those events that not older than given period to make sure we are
    // not grabbing GBs of data accidentally.
    final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;

    // Flag for detecting gaps between events.
    final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);

    IgnitePredicate<Event> p =
        new IgnitePredicate<Event>() {
          /** */
          private static final long serialVersionUID = 0L;

          @Override
          public boolean apply(Event e) {
            // Detects that events were lost.
            if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);

            // Retains events by lastOrder, period and type.
            return e.localOrder() > lastOrder
                && e.timestamp() > notOlderThan
                && F.contains(evtTypes, e.type());
          }
        };

    Collection<Event> evts = ignite.events().localQuery(p);

    // Update latest order in node local, if not empty.
    if (!evts.isEmpty()) {
      Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);

      nl.put(evtOrderKey, maxEvt.localOrder());
    }

    // Update throttle counter.
    if (!lastFound.get())
      nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);

    boolean lost = !lastFound.get() && throttle == 0;

    Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));

    if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));

    for (Event e : evts) {
      VisorGridEvent visorEvt = evtMapper.apply(e);

      if (visorEvt != null) res.add(visorEvt);
    }

    return res;
  }
Пример #8
0
  /** Simple test to check exceptions and other cases */
  @SuppressWarnings("unchecked")
  static void testExceptions(Path dir) throws IOException {
    System.out.println("-- Exceptions and other simple tests --");

    WatchService watcher = FileSystems.getDefault().newWatchService();
    try {

      // Poll tests

      WatchKey key;
      System.out.println("poll...");
      key = watcher.poll();
      if (key != null) throw new RuntimeException("no keys registered");

      System.out.println("poll with timeout...");
      try {
        long start = System.currentTimeMillis();
        key = watcher.poll(3000, TimeUnit.MILLISECONDS);
        if (key != null) throw new RuntimeException("no keys registered");
        long waited = System.currentTimeMillis() - start;
        if (waited < 2900) throw new RuntimeException("poll was too short");
      } catch (InterruptedException x) {
        throw new RuntimeException(x);
      }

      // IllegalArgumentException
      System.out.println("IllegalArgumentException tests...");
      try {
        dir.register(watcher, new WatchEvent.Kind<?>[] {});
        throw new RuntimeException("IllegalArgumentException not thrown");
      } catch (IllegalArgumentException x) {
      }
      try {
        // OVERFLOW is ignored so this is equivalent to the empty set
        dir.register(watcher, new WatchEvent.Kind<?>[] {OVERFLOW});
        throw new RuntimeException("IllegalArgumentException not thrown");
      } catch (IllegalArgumentException x) {
      }

      // UnsupportedOperationException
      try {
        dir.register(
            watcher,
            new WatchEvent.Kind<?>[] {
              new WatchEvent.Kind<Object>() {
                @Override
                public String name() {
                  return "custom";
                }

                @Override
                public Class<Object> type() {
                  return Object.class;
                }
              }
            });
      } catch (UnsupportedOperationException x) {
      }
      try {
        dir.register(
            watcher,
            new WatchEvent.Kind<?>[] {ENTRY_CREATE},
            new WatchEvent.Modifier() {
              @Override
              public String name() {
                return "custom";
              }
            });
        throw new RuntimeException("UnsupportedOperationException not thrown");
      } catch (UnsupportedOperationException x) {
      }

      // NullPointerException
      System.out.println("NullPointerException tests...");
      try {
        dir.register(null, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
        throw new RuntimeException("NullPointerException not thrown");
      } catch (NullPointerException x) {
      }
      try {
        dir.register(watcher, new WatchEvent.Kind<?>[] {null});
        throw new RuntimeException("NullPointerException not thrown");
      } catch (NullPointerException x) {
      }
      try {
        dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE}, (WatchEvent.Modifier) null);
        throw new RuntimeException("NullPointerException not thrown");
      } catch (NullPointerException x) {
      }
    } finally {
      watcher.close();
    }

    // -- ClosedWatchServiceException --

    System.out.println("ClosedWatchServiceException tests...");

    try {
      watcher.poll();
      throw new RuntimeException("ClosedWatchServiceException not thrown");
    } catch (ClosedWatchServiceException x) {
    }

    // assume that poll throws exception immediately
    long start = System.currentTimeMillis();
    try {
      watcher.poll(10000, TimeUnit.MILLISECONDS);
      throw new RuntimeException("ClosedWatchServiceException not thrown");
    } catch (InterruptedException x) {
      throw new RuntimeException(x);
    } catch (ClosedWatchServiceException x) {
      long waited = System.currentTimeMillis() - start;
      if (waited > 5000) throw new RuntimeException("poll was too long");
    }

    try {
      watcher.take();
      throw new RuntimeException("ClosedWatchServiceException not thrown");
    } catch (InterruptedException x) {
      throw new RuntimeException(x);
    } catch (ClosedWatchServiceException x) {
    }

    try {
      dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
      throw new RuntimeException("ClosedWatchServiceException not thrown");
    } catch (ClosedWatchServiceException x) {
    }

    System.out.println("OKAY");
  }