public void run() {
int baseKey = (index * keyCount) % keys.size();
for (int i = 0; i < requests; i++) {
int keyIndex = (baseKey + (i % keyCount)) % keys.size();
String key = keys.get(keyIndex);
Transaction txn = fac.create();
long start = System.currentTimeMillis();
try {
txn.begin();
byte[] value = txn.read(key);
txn.commit();
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] " + key + ": " + new String(value));
}
success++;
} catch (Exception e) {
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
}
failure++;
}
}
}
public void run() {
int baseKey = index;
for (int i = 0; i < requests; i++) {
int keyIndex = (baseKey + i);
String key = keys.get(keyIndex);
Transaction txn = fac.create();
long start = System.currentTimeMillis();
try {
txn.begin();
String value = "random_value_" + index + "_" + i;
txn.write(key, value.getBytes());
txn.commit();
timeElapsed += (System.currentTimeMillis() - start);
if (!silent.get()) {
System.out.println("[Thread-" + index + "] " + key + ": " + value);
}
success++;
} catch (TransactionException e) {
timeElapsed += (System.currentTimeMillis() - start);
if (!silent.get()) {
System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
}
failure++;
}
}
}
public static void main(String[] args) {
long start = System.currentTimeMillis();
FeatureExtractorSimple fe = new FeatureExtractorSimple(args);
fe.run();
long end = System.currentTimeMillis();
Logger.log("processing completed in " + (end - start) / 1000 + " sec");
Logger.close();
System.out.println("processing completed in " + (end - start) / 1000 + " sec");
}
private static void loadData(TransactionFactory fac, int numKeys, int concurrency) {
List<Future> futures = new ArrayList<Future>();
final List<String> keys = new ArrayList<String>();
for (int i = 0; i < numKeys; i++) {
keys.add("row" + i);
}
int success = 0;
int failure = 0;
// Use concurrency threads
int requestsPerIndex = numKeys / concurrency;
// This is to ensure you always load at least the keys you need
if (numKeys % concurrency != 0) {
requestsPerIndex++;
}
IndexedWriteWorker[] workers = new IndexedWriteWorker[numKeys];
int index = 0;
for (int i = 0; i < concurrency; i++) {
workers[i] = new IndexedWriteWorker(index, fac, requestsPerIndex, numKeys, keys);
index += requestsPerIndex;
}
long start = System.currentTimeMillis();
for (int i = 0; i < concurrency; i++) {
futures.add(exec.submit(workers[i]));
}
long time = 0L;
for (int i = 0; i < concurrency; i++) {
try {
futures.get(i).get();
} catch (Exception ignored) {
}
success += workers[i].success;
failure += workers[i].failure;
time += workers[i].timeElapsed;
}
long end = System.currentTimeMillis();
int total = success + failure;
System.out.println("\nSuccessful: " + success + "/" + total);
System.out.println("Failed: " + failure + "/" + total);
System.out.println("Time elapsed: " + (end - start) + "ms");
System.out.println("Throughput: " + total / ((end - start) / 1000.0) + " TPS");
System.out.println("Average latency: " + time / (double) total + "ms");
}
public void run() {
long start = System.currentTimeMillis();
combineRequire(require);
ArrayList<String> combinedFiles = new ArrayList<String>();
StringBuilder finalCodes = new StringBuilder();
for (Module m : processedModules) {
combinedFiles.add(m.getName());
finalCodes.append(m.getCode());
}
String re =
"/*\n Combined processedModules by KISSY Module Compiler: \n\n "
+ ArrayUtils.join(combinedFiles.toArray(new String[combinedFiles.size()]), "\n ")
+ "\n*/\n\n"
+ finalCodes.toString();
if (output != null) {
FileUtils.outputContent(re, output, outputEncoding);
System.out.println("success generated: " + output);
} else {
System.out.println(re);
}
if (outputDependency != null && dependencies.size() != 0) {
String allRs = "";
for (String r : dependencies) {
if (r.startsWith("#")) {
allRs += "," + r.substring(1);
} else {
allRs += ",'" + r + "'";
}
}
re = DEP_PREFIX + "'" + require + "': {requires: [" + allRs.substring(1) + "]}" + DEP_SUFFIX;
FileUtils.outputContent(re, outputDependency, outputEncoding);
System.out.println("success generated: " + outputDependency);
}
System.out.print("duration: " + (System.currentTimeMillis() - start));
}
public Q2(String[] args) {
super();
this.args = args;
startTime = System.currentTimeMillis();
instanceId = UUID.randomUUID();
parseCmdLine(args);
libDir = new File(deployDir, "lib");
dirMap = new TreeMap();
deployDir.mkdirs();
mainClassLoader = Thread.currentThread().getContextClassLoader();
}
/**
* returns the policy object from the policyObject option
*
* @return the policy object to be processed
*/
public void loadPO() {
if (genProps.getProperty("newPolicyLoc", null) == null)
System.err.println("newPolLoc == null in gio:loadPO");
File pLoc = new File(genProps.getProperty("newPolicyLoc", null));
if (!pLoc.exists()) {
System.err.println("no file found at p3p policy location specified by the new policy option");
System.exit(1);
}
po = (new P3PParser()).parse(pLoc.getAbsolutePath());
// TODO make sure that the context is parsed if avaliable
if (po.getContext().getDomain() == null) {
po.setContext(
new Context(
new Date(System.currentTimeMillis()),
new Date(System.currentTimeMillis()),
genProps.getProperty("newPolicyLoc")));
}
}
private static void randomBlindWriteTransactions(
final TransactionFactory fac, int c, int r, int k, int t) {
List<Future> futures = new ArrayList<Future>();
final List<String> keys = new ArrayList<String>();
for (int i = 0; i < t; i++) {
keys.add("KEY" + i);
}
int success = 0;
int failure = 0;
IndexedWriteWorker[] workers = new IndexedWriteWorker[c];
for (int i = 0; i < c; i++) {
workers[i] = new IndexedWriteWorker(i, fac, r, k, keys);
}
long start = System.currentTimeMillis();
for (int i = 0; i < c; i++) {
futures.add(exec.submit(workers[i]));
}
long time = 0L;
for (int i = 0; i < c; i++) {
try {
futures.get(i).get();
} catch (Exception ignored) {
}
success += workers[i].success;
failure += workers[i].failure;
time += workers[i].timeElapsed;
}
long end = System.currentTimeMillis();
int total = success + failure;
System.out.println("\nSuccessful: " + success + "/" + total);
System.out.println("Failed: " + failure + "/" + total);
System.out.println("Time elapsed: " + (end - start) + "ms");
System.out.println("Throughput: " + total / ((end - start) / 1000.0) + " TPS");
System.out.println("Average latency: " + time / (double) total + "ms");
}
public static void main(String[] args) throws Exception {
long t0 = System.currentTimeMillis();
final String OPT_JAR_FILE = "aijar";
final String OPT_CLASSNAME = "ainame";
Options options =
new Options()
.addOption(
Option.builder(OPT_JAR_FILE)
.hasArg()
.argName("jarfile")
.required()
.desc("The path to the jar file")
.build())
.addOption(
Option.builder(OPT_CLASSNAME)
.hasArg()
.argName("classname")
.required()
.desc("The full classpath of the Ai class")
.build())
.addOption(timeOpt);
DraughtStateParser.addOptions(options);
HelpFormatter formatter = new HelpFormatter();
CommandLine line = new DefaultParser().parse(options, args);
DraughtsPlayer player;
Integer time;
DraughtsState draughtsState;
try {
File aiJar = new File(line.getOptionValue(OPT_JAR_FILE));
if (!aiJar.canRead() || !aiJar.isFile()) {
throw new ParseException("Could not read from aijar");
}
String className = line.getOptionValue(OPT_CLASSNAME);
player = Main.getFromClass(aiJar, className);
time = Integer.parseInt(line.getOptionValue(OPT_TIME));
draughtsState = DraughtStateParser.parseDraughtsState(line);
} catch (JSONException e) {
formatter.printHelp("java -jar wtcl.jar", options);
System.err.println(
"JSON Exception on string " + line.getOptionValue(DraughtStateParser.OPT_BOARD));
throw e;
} catch (Exception e) {
formatter.printHelp("java -jar wtcl.jar", options);
throw e;
}
execute(player, draughtsState, time, t0);
}
public boolean printProgress() {
boolean done = true;
StringBuilder sb = new StringBuilder();
sb.append("\rprogress: ");
long totalProgress = 0;
long totalSize = 0;
for (Map.Entry<InetAddress, Collection<PendingFile>> entry : filesByHost.entrySet()) {
long progress = 0;
long size = 0;
int completed = 0;
Collection<PendingFile> pendings = entry.getValue();
for (PendingFile f : pendings) {
progress += f.progress;
size += f.size;
if (f.progress == f.size) completed++;
}
totalProgress += progress;
totalSize += size;
if (completed != pendings.size()) done = false;
sb.append("[").append(entry.getKey());
sb.append(" ").append(completed).append("/").append(pendings.size());
sb.append(" (").append(size == 0 ? 100L : progress * 100L / size).append(")] ");
}
long time = System.currentTimeMillis();
long deltaTime = time - lastTime;
lastTime = time;
long deltaProgress = totalProgress - lastProgress;
lastProgress = totalProgress;
sb.append("[total: ")
.append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
.append(" - ");
sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
sb.append(" (avg: ").append(mbPerSec(totalProgress, time - startTime)).append("MB/s)]");
;
System.out.print(sb.toString());
return done;
}
public AeronLatencyUnderLoadPublisher(final String[] args) {
try {
parseArgs(args);
} catch (final ParseException e) {
throw new RuntimeException(e);
}
final Aeron.Context ctx = new Aeron.Context().newImageHandler(this::imageHandler);
fragmentHandler = new FragmentAssembler(this::msgHandler);
final Aeron aeron = Aeron.connect(ctx);
System.out.println("Reflect: " + reflectChannel + " Pub: " + pubChannel);
pub = aeron.addPublication(pubChannel, pubStreamId);
sub = aeron.addSubscription(reflectChannel, subStreamId);
imageLatch = new CountDownLatch(1);
final IdleStrategy idle = new BusySpinIdleStrategy();
bufferClaim = new BufferClaim();
final List<RateControllerInterval> intervals = new ArrayList<>();
intervals.add(new MessagesAtMessagesPerSecondInterval(100, 10));
intervals.add(new MessagesAtMessagesPerSecondInterval(1000, 100));
intervals.add(new MessagesAtMessagesPerSecondInterval(10000, 1000));
intervals.add(new MessagesAtMessagesPerSecondInterval(100000, 10000));
intervals.add(new MessagesAtMessagesPerSecondInterval(1000000, 100000));
intervals.add(new MessagesAtMessagesPerSecondInterval(10000000, 1000000));
intervals.add(new MessagesAtMessagesPerSecondInterval(30000000, 3000000));
buffer = new UnsafeBuffer(ByteBuffer.allocateDirect(msgLen));
msgCount = 0;
RateController rateController = null;
try {
rateController = new RateController(this, intervals);
} catch (final Exception ex) {
throw new RuntimeException(ex);
}
final Runnable task =
() -> {
while (running) {
while (sub.poll(fragmentHandler, 1) <= 0 && running) {}
}
System.out.println("Done");
};
final Thread subThread = new Thread(task);
subThread.start();
try {
imageLatch.await();
} catch (final InterruptedException e) {
e.printStackTrace();
}
final int warmUpMsgs = 100000;
for (int i = 0; i < warmUpMsgs; i++) {
while (pub.tryClaim(buffer.capacity(), bufferClaim) < 0L) {
idle.idle(1);
}
final MutableDirectBuffer buffer = bufferClaim.buffer();
final int offset = bufferClaim.offset();
buffer.putByte(offset, (byte) 'w');
bufferClaim.commit();
}
try {
Thread.sleep(1000);
} catch (final InterruptedException e) {
e.printStackTrace();
}
System.out.println("warmup msgs received: " + warmups);
final int start = (int) System.currentTimeMillis();
while (rateController.next()) {}
final int total = (int) (System.currentTimeMillis() - start) / 1000;
buffer.putByte(0, (byte) 'q');
while (pub.offer(buffer, 0, buffer.capacity()) < 0L) {
idle.idle(0);
}
System.out.println("Duration: " + total + " seconds");
try {
Thread.sleep(1000);
} catch (final InterruptedException e) {
e.printStackTrace();
}
running = false;
try {
subThread.join();
} catch (final InterruptedException e) {
e.printStackTrace();
}
aeron.close();
try {
computeStats();
} catch (IOException e) {
e.printStackTrace();
}
}
public long getUptime() {
return System.currentTimeMillis() - startTime;
}
/** loads [additional] policies from commandline (either -p or -f) */
private void loadCLPolicies() {
// we already checked to make sure we have one of the options avaliable
File pLoc = null;
PolicyObject p = null;
if (genProps.getProperty("p3pLocation", null) != null) {
pLoc = new File(genProps.getProperty("p3pLocation"));
if (!pLoc.exists()) {
System.err.println(
"no file found at p3p policy location specified by the -p3p option: "
+ genProps.getProperty("p3pLocation"));
System.err.println("current location is " + System.getProperty("user.dir"));
System.exit(1);
}
try {
p = (new P3PParser()).parse(pLoc.getAbsolutePath());
if (p.getContextDomain() == null) {
p.setContext(
new Context(
new Date(System.currentTimeMillis()),
new Date(System.currentTimeMillis()),
genProps.getProperty("p3pLocation")));
}
pdb.addPolicy(p);
} catch (Exception e) {
System.err.println("Error with parsing or database");
e.printStackTrace();
// System.exit(5);
}
}
if (genProps.getProperty("p3pDirLocation", null) != null) {
pLoc = new File(genProps.getProperty("p3pDirLocation"));
File[] pfiles = pLoc.listFiles();
// System.err.println("pfiles for p3pDirLocation: "+pfiles);
for (int i = 0; i < pfiles.length; i++) {
pLoc = (pfiles[i]);
if (!pLoc.exists()) {
System.err.println(
"no file found at p3p policy location specified by the -p3pDirLocation option, "
+ genProps.getProperty("p3pDirLocation"));
System.exit(1);
}
try {
p = (new P3PParser()).parse(pLoc.getAbsolutePath());
if (p.getContext().getDomain() == null) {
p.setContext(
new Context(
new Date(System.currentTimeMillis()),
new Date(System.currentTimeMillis()),
pfiles[i].getAbsolutePath()));
}
pdb.addPolicy(p);
} catch (Exception e) {
System.err.println("Einar needs to fix this parsing error.");
e.printStackTrace();
}
}
}
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public void start() {
startTime = System.currentTimeMillis();
}