Ejemplo n.º 1
0
 public void run() {
   int baseKey = (index * keyCount) % keys.size();
   for (int i = 0; i < requests; i++) {
     int keyIndex = (baseKey + (i % keyCount)) % keys.size();
     String key = keys.get(keyIndex);
     Transaction txn = fac.create();
     long start = System.currentTimeMillis();
     try {
       txn.begin();
       byte[] value = txn.read(key);
       txn.commit();
       timeElapsed += System.currentTimeMillis() - start;
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] " + key + ": " + new String(value));
       }
       success++;
     } catch (Exception e) {
       timeElapsed += System.currentTimeMillis() - start;
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
       }
       failure++;
     }
   }
 }
Ejemplo n.º 2
0
 public void run() {
   int baseKey = index;
   for (int i = 0; i < requests; i++) {
     int keyIndex = (baseKey + i);
     String key = keys.get(keyIndex);
     Transaction txn = fac.create();
     long start = System.currentTimeMillis();
     try {
       txn.begin();
       String value = "random_value_" + index + "_" + i;
       txn.write(key, value.getBytes());
       txn.commit();
       timeElapsed += (System.currentTimeMillis() - start);
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] " + key + ": " + value);
       }
       success++;
     } catch (TransactionException e) {
       timeElapsed += (System.currentTimeMillis() - start);
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
       }
       failure++;
     }
   }
 }
Ejemplo n.º 3
0
  public static void main(String[] args) {
    long start = System.currentTimeMillis();
    FeatureExtractorSimple fe = new FeatureExtractorSimple(args);

    fe.run();
    long end = System.currentTimeMillis();
    Logger.log("processing completed in " + (end - start) / 1000 + " sec");
    Logger.close();
    System.out.println("processing completed in " + (end - start) / 1000 + " sec");
  }
Ejemplo n.º 4
0
  private static void loadData(TransactionFactory fac, int numKeys, int concurrency) {
    List<Future> futures = new ArrayList<Future>();

    final List<String> keys = new ArrayList<String>();
    for (int i = 0; i < numKeys; i++) {
      keys.add("row" + i);
    }

    int success = 0;
    int failure = 0;
    // Use concurrency threads
    int requestsPerIndex = numKeys / concurrency;
    // This is to ensure you always load at least the keys you need
    if (numKeys % concurrency != 0) {
      requestsPerIndex++;
    }
    IndexedWriteWorker[] workers = new IndexedWriteWorker[numKeys];
    int index = 0;
    for (int i = 0; i < concurrency; i++) {
      workers[i] = new IndexedWriteWorker(index, fac, requestsPerIndex, numKeys, keys);
      index += requestsPerIndex;
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < concurrency; i++) {
      futures.add(exec.submit(workers[i]));
    }

    long time = 0L;
    for (int i = 0; i < concurrency; i++) {
      try {
        futures.get(i).get();
      } catch (Exception ignored) {
      }
      success += workers[i].success;
      failure += workers[i].failure;
      time += workers[i].timeElapsed;
    }
    long end = System.currentTimeMillis();

    int total = success + failure;
    System.out.println("\nSuccessful: " + success + "/" + total);
    System.out.println("Failed: " + failure + "/" + total);
    System.out.println("Time elapsed: " + (end - start) + "ms");
    System.out.println("Throughput: " + total / ((end - start) / 1000.0) + " TPS");
    System.out.println("Average latency: " + time / (double) total + "ms");
  }
Ejemplo n.º 5
0
  public void run() {
    long start = System.currentTimeMillis();

    combineRequire(require);

    ArrayList<String> combinedFiles = new ArrayList<String>();
    StringBuilder finalCodes = new StringBuilder();

    for (Module m : processedModules) {
      combinedFiles.add(m.getName());
      finalCodes.append(m.getCode());
    }

    String re =
        "/*\n Combined processedModules by KISSY Module Compiler: \n\n "
            + ArrayUtils.join(combinedFiles.toArray(new String[combinedFiles.size()]), "\n ")
            + "\n*/\n\n"
            + finalCodes.toString();

    if (output != null) {
      FileUtils.outputContent(re, output, outputEncoding);
      System.out.println("success generated: " + output);
    } else {
      System.out.println(re);
    }

    if (outputDependency != null && dependencies.size() != 0) {
      String allRs = "";
      for (String r : dependencies) {
        if (r.startsWith("#")) {
          allRs += "," + r.substring(1);
        } else {
          allRs += ",'" + r + "'";
        }
      }
      re = DEP_PREFIX + "'" + require + "': {requires: [" + allRs.substring(1) + "]}" + DEP_SUFFIX;
      FileUtils.outputContent(re, outputDependency, outputEncoding);
      System.out.println("success generated: " + outputDependency);
    }

    System.out.print("duration: " + (System.currentTimeMillis() - start));
  }
Ejemplo n.º 6
0
 public Q2(String[] args) {
   super();
   this.args = args;
   startTime = System.currentTimeMillis();
   instanceId = UUID.randomUUID();
   parseCmdLine(args);
   libDir = new File(deployDir, "lib");
   dirMap = new TreeMap();
   deployDir.mkdirs();
   mainClassLoader = Thread.currentThread().getContextClassLoader();
 }
Ejemplo n.º 7
0
  /**
   * returns the policy object from the policyObject option
   *
   * @return the policy object to be processed
   */
  public void loadPO() {

    if (genProps.getProperty("newPolicyLoc", null) == null)
      System.err.println("newPolLoc == null in gio:loadPO");
    File pLoc = new File(genProps.getProperty("newPolicyLoc", null));
    if (!pLoc.exists()) {
      System.err.println("no file found at p3p policy location specified by the new policy option");
      System.exit(1);
    }

    po = (new P3PParser()).parse(pLoc.getAbsolutePath());
    // TODO make sure that the context is parsed if avaliable
    if (po.getContext().getDomain() == null) {
      po.setContext(
          new Context(
              new Date(System.currentTimeMillis()),
              new Date(System.currentTimeMillis()),
              genProps.getProperty("newPolicyLoc")));
    }
  }
Ejemplo n.º 8
0
  private static void randomBlindWriteTransactions(
      final TransactionFactory fac, int c, int r, int k, int t) {
    List<Future> futures = new ArrayList<Future>();

    final List<String> keys = new ArrayList<String>();
    for (int i = 0; i < t; i++) {
      keys.add("KEY" + i);
    }

    int success = 0;
    int failure = 0;
    IndexedWriteWorker[] workers = new IndexedWriteWorker[c];
    for (int i = 0; i < c; i++) {
      workers[i] = new IndexedWriteWorker(i, fac, r, k, keys);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < c; i++) {
      futures.add(exec.submit(workers[i]));
    }

    long time = 0L;
    for (int i = 0; i < c; i++) {
      try {
        futures.get(i).get();
      } catch (Exception ignored) {
      }
      success += workers[i].success;
      failure += workers[i].failure;
      time += workers[i].timeElapsed;
    }
    long end = System.currentTimeMillis();

    int total = success + failure;
    System.out.println("\nSuccessful: " + success + "/" + total);
    System.out.println("Failed: " + failure + "/" + total);
    System.out.println("Time elapsed: " + (end - start) + "ms");
    System.out.println("Throughput: " + total / ((end - start) / 1000.0) + " TPS");
    System.out.println("Average latency: " + time / (double) total + "ms");
  }
Ejemplo n.º 9
0
  public static void main(String[] args) throws Exception {
    long t0 = System.currentTimeMillis();

    final String OPT_JAR_FILE = "aijar";
    final String OPT_CLASSNAME = "ainame";

    Options options =
        new Options()
            .addOption(
                Option.builder(OPT_JAR_FILE)
                    .hasArg()
                    .argName("jarfile")
                    .required()
                    .desc("The path to the jar file")
                    .build())
            .addOption(
                Option.builder(OPT_CLASSNAME)
                    .hasArg()
                    .argName("classname")
                    .required()
                    .desc("The full classpath of the Ai class")
                    .build())
            .addOption(timeOpt);
    DraughtStateParser.addOptions(options);
    HelpFormatter formatter = new HelpFormatter();
    CommandLine line = new DefaultParser().parse(options, args);

    DraughtsPlayer player;
    Integer time;
    DraughtsState draughtsState;
    try {
      File aiJar = new File(line.getOptionValue(OPT_JAR_FILE));
      if (!aiJar.canRead() || !aiJar.isFile()) {
        throw new ParseException("Could not read from aijar");
      }
      String className = line.getOptionValue(OPT_CLASSNAME);
      player = Main.getFromClass(aiJar, className);
      time = Integer.parseInt(line.getOptionValue(OPT_TIME));
      draughtsState = DraughtStateParser.parseDraughtsState(line);
    } catch (JSONException e) {
      formatter.printHelp("java -jar wtcl.jar", options);
      System.err.println(
          "JSON Exception on string " + line.getOptionValue(DraughtStateParser.OPT_BOARD));
      throw e;
    } catch (Exception e) {
      formatter.printHelp("java -jar wtcl.jar", options);
      throw e;
    }
    execute(player, draughtsState, time, t0);
  }
Ejemplo n.º 10
0
    public boolean printProgress() {
      boolean done = true;
      StringBuilder sb = new StringBuilder();
      sb.append("\rprogress: ");
      long totalProgress = 0;
      long totalSize = 0;
      for (Map.Entry<InetAddress, Collection<PendingFile>> entry : filesByHost.entrySet()) {
        long progress = 0;
        long size = 0;
        int completed = 0;
        Collection<PendingFile> pendings = entry.getValue();
        for (PendingFile f : pendings) {
          progress += f.progress;
          size += f.size;
          if (f.progress == f.size) completed++;
        }
        totalProgress += progress;
        totalSize += size;
        if (completed != pendings.size()) done = false;
        sb.append("[").append(entry.getKey());
        sb.append(" ").append(completed).append("/").append(pendings.size());
        sb.append(" (").append(size == 0 ? 100L : progress * 100L / size).append(")] ");
      }
      long time = System.currentTimeMillis();
      long deltaTime = time - lastTime;
      lastTime = time;
      long deltaProgress = totalProgress - lastProgress;
      lastProgress = totalProgress;

      sb.append("[total: ")
          .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
          .append(" - ");
      sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
      sb.append(" (avg: ").append(mbPerSec(totalProgress, time - startTime)).append("MB/s)]");
      ;
      System.out.print(sb.toString());
      return done;
    }
  public AeronLatencyUnderLoadPublisher(final String[] args) {
    try {
      parseArgs(args);
    } catch (final ParseException e) {
      throw new RuntimeException(e);
    }
    final Aeron.Context ctx = new Aeron.Context().newImageHandler(this::imageHandler);
    fragmentHandler = new FragmentAssembler(this::msgHandler);
    final Aeron aeron = Aeron.connect(ctx);
    System.out.println("Reflect: " + reflectChannel + " Pub: " + pubChannel);
    pub = aeron.addPublication(pubChannel, pubStreamId);
    sub = aeron.addSubscription(reflectChannel, subStreamId);
    imageLatch = new CountDownLatch(1);
    final IdleStrategy idle = new BusySpinIdleStrategy();
    bufferClaim = new BufferClaim();

    final List<RateControllerInterval> intervals = new ArrayList<>();
    intervals.add(new MessagesAtMessagesPerSecondInterval(100, 10));
    intervals.add(new MessagesAtMessagesPerSecondInterval(1000, 100));
    intervals.add(new MessagesAtMessagesPerSecondInterval(10000, 1000));
    intervals.add(new MessagesAtMessagesPerSecondInterval(100000, 10000));
    intervals.add(new MessagesAtMessagesPerSecondInterval(1000000, 100000));
    intervals.add(new MessagesAtMessagesPerSecondInterval(10000000, 1000000));
    intervals.add(new MessagesAtMessagesPerSecondInterval(30000000, 3000000));
    buffer = new UnsafeBuffer(ByteBuffer.allocateDirect(msgLen));
    msgCount = 0;

    RateController rateController = null;
    try {
      rateController = new RateController(this, intervals);
    } catch (final Exception ex) {
      throw new RuntimeException(ex);
    }

    final Runnable task =
        () -> {
          while (running) {
            while (sub.poll(fragmentHandler, 1) <= 0 && running) {}
          }
          System.out.println("Done");
        };
    final Thread subThread = new Thread(task);
    subThread.start();

    try {
      imageLatch.await();
    } catch (final InterruptedException e) {
      e.printStackTrace();
    }

    final int warmUpMsgs = 100000;
    for (int i = 0; i < warmUpMsgs; i++) {
      while (pub.tryClaim(buffer.capacity(), bufferClaim) < 0L) {
        idle.idle(1);
      }
      final MutableDirectBuffer buffer = bufferClaim.buffer();
      final int offset = bufferClaim.offset();
      buffer.putByte(offset, (byte) 'w');
      bufferClaim.commit();
    }
    try {
      Thread.sleep(1000);
    } catch (final InterruptedException e) {
      e.printStackTrace();
    }
    System.out.println("warmup msgs received: " + warmups);
    final int start = (int) System.currentTimeMillis();
    while (rateController.next()) {}

    final int total = (int) (System.currentTimeMillis() - start) / 1000;
    buffer.putByte(0, (byte) 'q');

    while (pub.offer(buffer, 0, buffer.capacity()) < 0L) {
      idle.idle(0);
    }

    System.out.println("Duration: " + total + " seconds");
    try {
      Thread.sleep(1000);
    } catch (final InterruptedException e) {
      e.printStackTrace();
    }

    running = false;

    try {
      subThread.join();
    } catch (final InterruptedException e) {
      e.printStackTrace();
    }

    aeron.close();

    try {
      computeStats();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Ejemplo n.º 12
0
 public long getUptime() {
   return System.currentTimeMillis() - startTime;
 }
Ejemplo n.º 13
0
  /** loads [additional] policies from commandline (either -p or -f) */
  private void loadCLPolicies() {
    // we already checked to make sure we have one of the options avaliable
    File pLoc = null;
    PolicyObject p = null;

    if (genProps.getProperty("p3pLocation", null) != null) {
      pLoc = new File(genProps.getProperty("p3pLocation"));
      if (!pLoc.exists()) {
        System.err.println(
            "no file found at p3p policy location specified by the -p3p option: "
                + genProps.getProperty("p3pLocation"));
        System.err.println("current location is " + System.getProperty("user.dir"));
        System.exit(1);
      }
      try {
        p = (new P3PParser()).parse(pLoc.getAbsolutePath());
        if (p.getContextDomain() == null) {
          p.setContext(
              new Context(
                  new Date(System.currentTimeMillis()),
                  new Date(System.currentTimeMillis()),
                  genProps.getProperty("p3pLocation")));
        }
        pdb.addPolicy(p);
      } catch (Exception e) {
        System.err.println("Error with parsing or database");
        e.printStackTrace();
        // System.exit(5);
      }
    }
    if (genProps.getProperty("p3pDirLocation", null) != null) {
      pLoc = new File(genProps.getProperty("p3pDirLocation"));
      File[] pfiles = pLoc.listFiles();

      // System.err.println("pfiles for p3pDirLocation: "+pfiles);
      for (int i = 0; i < pfiles.length; i++) {

        pLoc = (pfiles[i]);
        if (!pLoc.exists()) {
          System.err.println(
              "no file found at p3p policy location specified by the -p3pDirLocation option, "
                  + genProps.getProperty("p3pDirLocation"));
          System.exit(1);
        }
        try {

          p = (new P3PParser()).parse(pLoc.getAbsolutePath());
          if (p.getContext().getDomain() == null) {
            p.setContext(
                new Context(
                    new Date(System.currentTimeMillis()),
                    new Date(System.currentTimeMillis()),
                    pfiles[i].getAbsolutePath()));
          }
          pdb.addPolicy(p);
        } catch (Exception e) {
          System.err.println("Einar needs to fix this parsing error.");
          e.printStackTrace();
        }
      }
    }
  }
Ejemplo n.º 14
0
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas();

      String ksName = null;
      String cfName = null;
      Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
      for (String filename : options.filenames) {
        File file = new File(filename);
        if (!file.exists()) {
          System.out.println("Skipping inexisting file " + file);
          continue;
        }

        Pair<Descriptor, Component> pair =
            SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
        if (pair == null) {
          System.out.println("Skipping non sstable file " + file);
          continue;
        }
        Descriptor desc = pair.left;

        if (ksName == null) ksName = desc.ksname;
        else if (!ksName.equals(desc.ksname))
          throw new IllegalArgumentException("All sstables must be part of the same keyspace");

        if (cfName == null) cfName = desc.cfname;
        else if (!cfName.equals(desc.cfname))
          throw new IllegalArgumentException("All sstables must be part of the same column family");

        Set<Component> components =
            new HashSet<Component>(
                Arrays.asList(
                    new Component[] {
                      Component.DATA,
                      Component.PRIMARY_INDEX,
                      Component.FILTER,
                      Component.COMPRESSION_INFO,
                      Component.STATS
                    }));

        Iterator<Component> iter = components.iterator();
        while (iter.hasNext()) {
          Component component = iter.next();
          if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
        }
        parsedFilenames.put(desc, components);
      }

      if (ksName == null || cfName == null) {
        System.err.println("No valid sstables to split");
        System.exit(1);
      }

      // Do not load sstables since they might be broken
      Table table = Table.openWithoutSSTables(ksName);
      ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);

      String snapshotName = "pre-split-" + System.currentTimeMillis();

      List<SSTableReader> sstables = new ArrayList<SSTableReader>();
      for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
          sstables.add(sstable);

          if (options.snapshot) {
            File snapshotDirectory =
                Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            sstable.createLinks(snapshotDirectory.getPath());
          }

        } catch (Exception e) {
          System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      if (options.snapshot)
        System.out.println(
            String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

      cfs.getDataTracker().markCompacting(sstables);
      for (SSTableReader sstable : sstables) {
        try {
          new SSTableSplitter(cfs, sstable, options.sizeInMB).split();

          // Remove the sstable
          sstable.markCompacted();
          sstable.releaseReference();
        } catch (Exception e) {
          System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      SSTableDeletingTask.waitForDeletions();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Ejemplo n.º 15
0
 public void start() {
   startTime = System.currentTimeMillis();
 }