public void start() {
if (watch.isRunning()) {
watch.reset().start();
} else {
watch.start();
}
}
/**
* DO NOT RUN!!!!
*
* @author Joshua Barlin (propoke24)
* @version 1
* @return Time between execution and interruption
* @deprecated Test Code
*/
@Deprecated
public static long timer() {
final Stopwatch stopwatch = Stopwatch.createUnstarted();
stopwatch.start();
stopwatch.stop();
return stopwatch.elapsed(TimeUnit.SECONDS);
}
@Override
public List<EndpointAffinity> getOperatorAffinity() {
watch.reset();
watch.start();
Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
endpointMap.put(ep.getAddress(), ep);
}
Map<DrillbitEndpoint, EndpointAffinity> affinityMap =
new HashMap<DrillbitEndpoint, EndpointAffinity>();
for (ServerName sn : regionsToScan.values()) {
DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
if (ep != null) {
EndpointAffinity affinity = affinityMap.get(ep);
if (affinity == null) {
affinityMap.put(ep, new EndpointAffinity(ep, 1));
} else {
affinity.addAffinity(1);
}
}
}
logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
return Lists.newArrayList(affinityMap.values());
}
private void schedule(int delay) {
currentDelay = delay;
watch = new Stopwatch();
watch.start();
taskId =
Bukkit.getScheduler()
.scheduleSyncDelayedTask(plugin, this, (long) delay * TICKS_PER_SECONDS);
}
@Test
public void testMillionsExe() {
Stopwatch stopwatch = Stopwatch.createStarted();
stopwatch.start();
int size = 1000 * 1000;
for (int i = 0; i < size; i++) {
cacheDemo.getUserWithoutAnno(i);
}
stopwatch.stop();
System.out.println(stopwatch.elapsed(TimeUnit.MILLISECONDS));
}
@Test
public void testAppend() {
MemTable mt = new MemTable();
Stopwatch sw = new Stopwatch();
sw.start();
for (int i = 0; i < 200000; i++) {
Message msg = getMessage(i);
mt.append(msg);
}
sw.stop();
System.out.println("ex time set:" + sw);
mt.getSnapShot();
}
/**
* Code for each 'client' to run.
*
* @param id
* @param c
* @param sharedConnection
* @throws IOException
*/
static void cycle(int id, final Configuration c, final HConnection sharedConnection)
throws IOException {
HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE);
table.setAutoFlushTo(false);
long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000);
long startTime = System.currentTimeMillis();
final int printInterval = 100000;
Random rd = new Random(id);
boolean get = c.getBoolean("hbase.test.do.gets", false);
try {
Stopwatch stopWatch = new Stopwatch();
stopWatch.start();
for (int i = 0; i < namespaceSpan; i++) {
byte[] b = format(rd.nextLong());
if (get) {
Get g = new Get(b);
table.get(g);
} else {
Put p = new Put(b);
p.add(HConstants.CATALOG_FAMILY, b, b);
table.put(p);
}
if (i % printInterval == 0) {
LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis());
stopWatch.reset();
stopWatch.start();
}
}
LOG.info(
"Finished a cycle putting "
+ namespaceSpan
+ " in "
+ (System.currentTimeMillis() - startTime)
+ "ms");
} finally {
table.close();
}
}
/**
* Attempt to kill a running task. If the task has not started running, it will not start. If it's
* already running, a kill request will be sent to it.
*
* <p>The AM will be informed about the task kill.
*/
public void killTask() {
if (!isCompleted.get()) {
if (!killInvoked.getAndSet(true)) {
synchronized (this) {
LOG.info(
"Kill task requested for id={}, taskRunnerSetup={}",
taskSpec.getTaskAttemptID(),
(taskRunner != null));
if (taskRunner != null) {
killtimerWatch.start();
LOG.info("Issuing kill to task {}", taskSpec.getTaskAttemptID());
boolean killed = taskRunner.killTask();
if (killed) {
// Sending a kill message to the AM right here. Don't need to wait for the task to
// complete.
LOG.info(
"Kill request for task {} completed. Informing AM", taskSpec.getTaskAttemptID());
reportTaskKilled();
} else {
LOG.info(
"Kill request for task {} did not complete because the task is already complete",
taskSpec.getTaskAttemptID());
}
shouldRunTask = false;
} else {
// If the task hasn't started, and it is killed - report back to the AM that the task
// has been killed.
LOG.debug("Reporting taskKilled for non-started fragment {}", getRequestId());
reportTaskKilled();
}
if (!isStarted.get()) {
// If the task hasn't started - inform about fragment completion immediately. It's
// possible for
// the callable to never run.
fragmentCompletionHanler.fragmentComplete(fragmentInfo);
this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort());
}
}
} else {
// This should not happen.
LOG.warn(
"Ignoring kill request for task {} since a previous kill request was processed",
taskSpec.getTaskAttemptID());
}
} else {
LOG.info(
"Ignoring kill request for task {} since it's already complete",
taskSpec.getTaskAttemptID());
}
}
private static List<int[]> getPopularTags(BookmarkReader reader, int sampleSize, int limit) {
timeString = "";
List<int[]> tags = new ArrayList<int[]>();
Stopwatch timer = new Stopwatch();
timer.start();
int[] tagIDs = getPopularTagList(reader, limit);
timer.stop();
long trainingTime = timer.elapsed(TimeUnit.MILLISECONDS);
timer = new Stopwatch();
timer.start();
for (int j = 0; j < sampleSize; j++) {
tags.add(tagIDs);
}
timer.stop();
long testTime = timer.elapsed(TimeUnit.MILLISECONDS);
timeString += ("Full training time: " + trainingTime + "\n");
timeString += ("Full test time: " + testTime + "\n");
timeString += ("Average test time: " + testTime / sampleSize) + "\n";
timeString += ("Total time: " + (trainingTime + testTime) + "\n");
return tags;
}
public static void main(String[] args) throws Exception {
// Thread.sleep(5000);
parserArgs(args);
Stopwatch stopwatch = new Stopwatch();
stopwatch.start();
startupProducer(fileName);
printResult(getEntryOrdering(), top, statisticsWord());
stopwatch.stop();
System.out.println("task elapsed time\t" + stopwatch.elapsed(TimeUnit.MILLISECONDS) + " ms");
executor.shutdown();
}
@Override
protected Collection<Commitment> getCommitments(String dep, String line) throws Exception {
Query dq = new TermQuery(new Term("responsibleDepartment", dep));
Query lq = new TermQuery(new Term("budgetLine", line));
BooleanQuery fq = new BooleanQuery();
if (departmentFirst) {
fq.add(dq, BooleanClause.Occur.MUST);
fq.add(lq, BooleanClause.Occur.MUST);
} else {
fq.add(lq, BooleanClause.Occur.MUST);
fq.add(dq, BooleanClause.Occur.MUST);
}
keySearch.start();
List<Object> keys = indexSearcher.search(commitmentRegion.getFullPath(), fq);
keySearch.stop();
hashGet.start();
Map<String, Commitment> result = commitmentRegion.getAll(keys);
hashGet.stop();
return result.values();
}
void transitionService(Service service, Service.State from, Service.State to) {
Preconditions.checkNotNull(service);
Preconditions.checkArgument(from != to);
this.monitor.enter();
try {
this.transitioned = true;
if (!this.ready) {
return;
}
Preconditions.checkState(
this.servicesByState.remove(from, service),
"Service %s not at the expected location in the state map %s",
new Object[] {service, from});
Preconditions.checkState(
this.servicesByState.put(to, service),
"Service %s in the state map unexpectedly at %s",
new Object[] {service, to});
Stopwatch stopwatch = (Stopwatch) this.startupTimers.get(service);
if (from == Service.State.NEW) {
stopwatch.start();
}
if ((to.compareTo(Service.State.RUNNING) >= 0) && (stopwatch.isRunning())) {
stopwatch.stop();
if (!(service instanceof ServiceManager.NoOpService)) {
ServiceManager.logger.log(
Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
}
}
if (to == Service.State.FAILED) {
fireFailedListeners(service);
}
if (this.states.count(Service.State.RUNNING) == this.numberOfServices) {
fireHealthyListeners();
} else if (this.states.count(Service.State.TERMINATED)
+ this.states.count(Service.State.FAILED)
== this.numberOfServices) {
fireStoppedListeners();
}
} finally {
this.monitor.leave();
executeListeners();
}
}
private void test(String description, int iterations, Runnable task) {
LOGGER.info("Running test: " + description);
long best = Long.MAX_VALUE;
Stopwatch stopwatch = new Stopwatch();
for (int i = 0; i < iterations; i++) {
stopwatch.start();
task.run();
stopwatch.stop();
long elapsed = stopwatch.elapsed(TimeUnit.MICROSECONDS);
best = Math.min(best, elapsed);
stopwatch.reset();
}
LOGGER.info("Finished test " + description + " in " + best + "µs");
}
public static void main(String[] args) {
Stopwatch watch = new Stopwatch();
watch.start();
GlydarBootstrap bootstrap = new GlydarBootstrap(args);
server = new GServer(bootstrap);
ParaGlydar.setServer(server);
serverThread = new Thread(server);
serverBootstrap = new ServerBootstrap();
serverBootstrap
.childHandler(new ProtocolInitializer())
.option(ChannelOption.TCP_NODELAY, true)
.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 32 * 1024)
.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 64 * 1024)
.group(new NioEventLoopGroup())
.channelFactory(
new ChannelFactory<ServerChannel>() {
@Override
public ServerChannel newChannel() {
return new NioServerSocketChannel();
}
})
.bind(new InetSocketAddress(server.getConfig().getPort()));
server.setUpWorlds();
try {
server.getPluginLoader().loadPlugins();
} catch (Exception exc) {
server.getLogger().warning(exc, "Error while loading plugins");
}
server.getLogger().info("Server ready on port {0}", server.getConfig().getPort());
server
.getLogger()
.info("This server is running {0} version {1}", server.getName(), server.getVersion());
watch.stop();
server.getLogger().info("Server started in {0}ms", watch.elapsed(TimeUnit.MILLISECONDS));
server.getCommandReader().start();
serverThread.start();
}
private void narrowByRule(ConstrainedTerm constrainedTerm, Rule rule) {
stopwatch.reset();
stopwatch.start();
constrainedTermResults = new ArrayList<ConstrainedTerm>();
SymbolicConstraint leftHandSideConstraint =
new SymbolicConstraint(constrainedTerm.termContext());
leftHandSideConstraint.addAll(rule.requires());
for (Variable variable : rule.freshVariables()) {
leftHandSideConstraint.add(variable, IntToken.fresh());
}
ConstrainedTerm leftHandSide =
new ConstrainedTerm(
rule.leftHandSide(),
rule.lookups().getSymbolicConstraint(constrainedTerm.termContext()),
leftHandSideConstraint,
constrainedTerm.termContext());
for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
constraint.addAll(rule.ensures());
/* rename rule variables in the constraints */
Map<Variable, Variable> freshSubstitution = constraint.rename(rule.variableSet());
Term result = rule.rightHandSide();
/* rename rule variables in the rule RHS */
result = result.substituteWithBinders(freshSubstitution, constrainedTerm.termContext());
/* apply the constraints substitution on the rule RHS */
result =
result.substituteWithBinders(constraint.substitution(), constrainedTerm.termContext());
/* evaluate pending functions in the rule RHS */
result = result.evaluate(constrainedTerm.termContext());
/* eliminate anonymous variables */
constraint.eliminateAnonymousVariables();
/* compute all results */
constrainedTermResults.add(
new ConstrainedTerm(result, constraint, constrainedTerm.termContext()));
}
stopwatch.stop();
}
private void testIteration() throws IOException {
for (int number = 1; number < 10_001; number *= 10) {
String jsonString = generateJajascriptJSON(number);
HttpRequest request =
HttpRequest.postBuilder("/jajascript/optimize")
.contentType(ContentTypes.APPLICATION_JSON, StandardCharsets.US_ASCII)
.body(jsonString)
.build();
Action action1 = new JajascriptAction();
Stopwatch stopwatch = new Stopwatch();
stopwatch.start();
HttpResponse response = action1.execute(request);
stopwatch.stop();
System.out.println(
"For " + number + ", result = " + stopwatch.elapsed(TimeUnit.MICROSECONDS) + "µs.");
System.out.println("Result = " + response.getBodyAsString(StandardCharsets.US_ASCII));
}
}
public void start() {
Stopwatch sw = new Stopwatch();
sw.start();
ServerCodecConfig codeConfig = new ServerCodecConfig(name, addr);
ThriftServerFramedCodec codec =
new ThriftServerFramedCodec(codeConfig, new TCompactProtocol.Factory());
server =
ServerBuilder.safeBuild(
svc,
ServerBuilder.get()
.codec(codec)
.name(name)
.maxConcurrentRequests(maxConcurrentRequests)
.bindTo(addr));
sw.stop();
logger.info(
String.format("building finagle server took %s ms", sw.elapsed(TimeUnit.MILLISECONDS)));
}
@Test
@Ignore
public void testParseParquetPhysicalPlanRemote() throws Exception {
DrillConfig config = DrillConfig.create();
try (DrillClient client = new DrillClient(config); ) {
client.connect();
ParquetResultsListener listener = new ParquetResultsListener();
Stopwatch watch = new Stopwatch();
watch.start();
client.runQuery(
org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
Resources.toString(Resources.getResource(fileName), Charsets.UTF_8),
listener);
System.out.println(
String.format(
"Got %d total records in %d seconds",
listener.await(), watch.elapsed(TimeUnit.SECONDS)));
client.close();
}
}
// apply rule by matching
private void rewriteByRule(Term term, Rule rule) {
stopwatch.reset();
stopwatch.start();
termResults = new ArrayList<Term>();
TermContext context = TermContext.of(definition);
ConstrainedTerm constrainedTerm = new ConstrainedTerm(term, context);
SymbolicConstraint leftHandSideConstraint = new SymbolicConstraint(context);
leftHandSideConstraint.addAll(rule.requires());
for (Variable variable : rule.freshVariables()) {
leftHandSideConstraint.add(variable, IntToken.fresh());
}
ConstrainedTerm leftHandSide =
new ConstrainedTerm(
rule.leftHandSide(),
rule.lookups().getSymbolicConstraint(context),
leftHandSideConstraint,
context);
for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
if (!constraint.isMatching(leftHandSide)) {
continue;
}
constraint.orientSubstitution(leftHandSide.variableSet());
Term result = rule.rightHandSide();
/* apply the constraints substitution on the rule RHS */
result = result.substituteAndEvaluate(constraint.substitution(), context);
/* compute all results */
termResults.add(result);
}
stopwatch.stop();
}
@Override
public void doFilterInternal(
final HttpServletRequest request,
final HttpServletResponse response,
final FilterChain filterChain)
throws IOException, ServletException {
if (LOG.isDebugEnabled()) {
final String requestDetails = buildRequestDetails(request);
if (LOG.isDebugEnabled()) {
LOG.debug(requestDetails + "Begin");
}
logCookies(request);
final ResponseWrapper wrappedResponse = new ResponseWrapper(response);
final Stopwatch stopwatch = Stopwatch.createUnstarted();
stopwatch.start();
try {
filterChain.doFilter(request, wrappedResponse);
} finally {
stopwatch.stop();
final int status = wrappedResponse.getStatus();
if (status != 0) {
LOG.debug(requestDetails + stopwatch.toString() + " (" + status + ")");
} else {
LOG.debug(requestDetails + stopwatch.toString());
}
}
return;
}
filterChain.doFilter(request, response);
}
public synchronized void scheduleRequest() {
if (closed || (future != null) || scheduled) {
return;
}
scheduled = true;
// start before scheduling to include error delay
errorStopwatch.start();
executor.schedule(
() -> {
try {
initiateRequest();
} catch (Throwable t) {
// should not happen, but be safe and fail the operator
clientCallback.clientFailed(HttpPageBufferClient.this, t);
}
},
errorDelayMillis,
TimeUnit.MILLISECONDS);
lastUpdate = DateTime.now();
requestsScheduled.incrementAndGet();
}
public static void startTimer(String name) {
Stopwatch stopwatch = Stopwatch.createUnstarted();
timers.put(name + Thread.currentThread().getId(), stopwatch);
stopwatch.start();
}
@Override
public void methodStarting() {
startNanos = componentStopwatch.elapsed(NANOSECONDS);
stopwatch.start();
}
private Integer mergeTranslations(
final Long sourceVersionId,
final Long targetVersionId,
final int batchStart,
final int batchLength,
final boolean useNewerTranslation,
final List<HLocale> supportedLocales)
throws Exception {
final Stopwatch stopwatch = Stopwatch.createUnstarted();
stopwatch.start();
List<HTextFlow[]> matches =
textFlowDAO.getSourceByMatchedContext(
sourceVersionId, targetVersionId, batchStart, batchLength);
Multimap<DocumentLocaleKey, TextFlowTargetStateChange> eventMap = HashMultimap.create();
Map<DocumentLocaleKey, Map<ContentState, Long>> docStatsMap = Maps.newHashMap();
Map<DocumentLocaleKey, Long> lastUpdatedTargetId = Maps.newHashMap();
;
for (HTextFlow[] results : matches) {
HTextFlow sourceTf = results[0];
HTextFlow targetTf = results[1];
boolean foundChange = false;
Map<Long, ContentState> localeContentStateMap = Maps.newHashMap();
for (HLocale hLocale : supportedLocales) {
HTextFlowTarget sourceTft = sourceTf.getTargets().get(hLocale.getId());
// only process translated state
if (sourceTft == null || !sourceTft.getState().isTranslated()) {
continue;
}
HTextFlowTarget targetTft = targetTf.getTargets().get(hLocale.getId());
if (targetTft == null) {
targetTft = new HTextFlowTarget(targetTf, hLocale);
targetTft.setVersionNum(0);
targetTf.getTargets().put(hLocale.getId(), targetTft);
}
if (MergeTranslationsServiceImpl.shouldMerge(sourceTft, targetTft, useNewerTranslation)) {
foundChange = true;
ContentState oldState = targetTft.getState();
localeContentStateMap.put(hLocale.getId(), oldState);
mergeTextFlowTarget(sourceTft, targetTft);
}
}
if (foundChange) {
translationStateCacheImpl.clearDocumentStatistics(targetTf.getDocument().getId());
textFlowDAO.makePersistent(targetTf);
textFlowDAO.flush();
for (Map.Entry<Long, ContentState> entry : localeContentStateMap.entrySet()) {
HTextFlowTarget updatedTarget = targetTf.getTargets().get(entry.getKey());
DocumentLocaleKey key =
new DocumentLocaleKey(
targetTf.getDocument().getId(), updatedTarget.getLocale().getLocaleId());
eventMap.put(
key,
new TextFlowTargetStateEvent.TextFlowTargetStateChange(
targetTf.getId(),
updatedTarget.getId(),
updatedTarget.getState(),
entry.getValue()));
lastUpdatedTargetId.put(key, updatedTarget.getId());
Map<ContentState, Long> contentStateDeltas =
docStatsMap.get(key) == null ? Maps.newHashMap() : docStatsMap.get(key);
DocStatsEvent.updateContentStateDeltas(
contentStateDeltas,
updatedTarget.getState(),
entry.getValue(),
targetTf.getWordCount());
docStatsMap.put(key, contentStateDeltas);
}
}
}
Long actorId = authenticatedAccount.getPerson().getId();
for (Map.Entry<DocumentLocaleKey, Collection<TextFlowTargetStateChange>> entry :
eventMap.asMap().entrySet()) {
TextFlowTargetStateEvent tftUpdatedEvent =
new TextFlowTargetStateEvent(
entry.getKey(), targetVersionId, actorId, ImmutableList.copyOf(entry.getValue()));
textFlowTargetStateEvent.fire(tftUpdatedEvent);
}
for (Map.Entry<DocumentLocaleKey, Map<ContentState, Long>> entry : docStatsMap.entrySet()) {
DocStatsEvent docEvent =
new DocStatsEvent(
entry.getKey(),
targetVersionId,
entry.getValue(),
lastUpdatedTargetId.get(entry.getKey()));
docStatsEvent.fire(docEvent);
}
stopwatch.stop();
log.info(
"Complete merge translations of {} in {}",
matches.size() * supportedLocales.size(),
stopwatch);
return matches.size() * supportedLocales.size();
}
/**
* Dame todos los paises
*
* <p>http://api.geonames.org/countryInfo?lang=es&username=gsantosgo&style=full
* http://api.geonames.org/search?featureCode=PCLI&featureCode=PCL&lang=es&username=gsantosgo&style=full
*
* @param args
*/
public static void main(String args[]) {
MyWebService.setUserName("gsantosgo");
System.out.println("======================================");
System.out.println(" Geonames Server: " + MyWebService.getGeoNamesServer());
System.out.println(" Geonames Server: " + MyWebService.getUserName());
/*
// Configuracion PROXY. Para conectarse correctamente este necesario configurar el PROXY.
System.setProperty("http.proxySet","true");
System.setProperty("http.proxyHost","10.14.79.204");
System.setProperty("http.proxyPort","8080");
System.setProperty("http.proxyUser","");
System.setProperty("http.proxyPassword","");
*/
Stopwatch stopwatch = new Stopwatch();
// Step 1. CountryInfo ======
stopwatch.start();
ToponymSearchCriteria toponymSearchCriteria = new ToponymSearchCriteria();
toponymSearchCriteria.setLanguage("es");
toponymSearchCriteria.setStyle(Style.FULL);
ToponymSearchResult toponymSearchResult = new ToponymSearchResult();
try {
toponymSearchResult = MyWebService.countryInfo(toponymSearchCriteria);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Número de Paises: " + toponymSearchResult.getTotalResultsCount());
List<Toponym> toponymList = toponymSearchResult.getToponyms();
// for (Toponym toponym : toponymList) {
// System.out.println(toponym);
// }
stopwatch.stop();
System.out.println(
String.format(
"Tiempo transcurrido en %d miliseconds: ",
stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));
stopwatch.reset();
stopwatch.start();
// Step 2. Country Search
ToponymSearchCriteria toponymCountrySearchCriteria = new ToponymSearchCriteria();
toponymCountrySearchCriteria.setLanguage("es");
toponymCountrySearchCriteria.setStyle(Style.FULL);
toponymCountrySearchCriteria.setFeatureClass(FeatureClass.A);
toponymCountrySearchCriteria.setFeatureCodes(
new String[] {"PCL", "PCLD", "PCLF", "PCLI", "PCLIX", "PCLS"});
ToponymSearchResult toponymCountrySearchResult = new ToponymSearchResult();
try {
toponymCountrySearchResult = MyWebService.search(toponymCountrySearchCriteria);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Número de Paises: " + toponymCountrySearchResult.getTotalResultsCount());
List<Toponym> toponymCountryList = toponymCountrySearchResult.getToponyms();
Map<Integer, Toponym> mapToponyms = Maps.newHashMap();
for (Toponym toponym : toponymCountryList) {
mapToponyms.put(toponym.getGeoNameId(), toponym);
}
int count = 0;
for (Toponym toponym : toponymList) {
Toponym mapToponym = mapToponyms.get(toponym.getGeoNameId());
if (mapToponym == null) {
count++;
}
}
System.out.println(" ==>" + count);
stopwatch.stop();
System.out.println(
String.format(
"Tiempo transcurrido en %d miliseconds: ",
stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));
}
/** @param incomingEndpoints */
@Override
public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
watch.reset();
watch.start();
final int numSlots = incomingEndpoints.size();
Preconditions.checkArgument(
numSlots <= regionsToScan.size(),
String.format(
"Incoming endpoints %d is greater than number of scan regions %d",
numSlots, regionsToScan.size()));
/*
* Minimum/Maximum number of assignment per slot
*/
final int minPerEndpointSlot = (int) Math.floor((double) regionsToScan.size() / numSlots);
final int maxPerEndpointSlot = (int) Math.ceil((double) regionsToScan.size() / numSlots);
/*
* initialize (endpoint index => HBaseSubScanSpec list) map
*/
endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);
/*
* another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list
*/
Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();
/*
* Initialize these two maps
*/
for (int i = 0; i < numSlots; ++i) {
endpointFragmentMapping.put(i, new ArrayList<HBaseSubScanSpec>(maxPerEndpointSlot));
String hostname = incomingEndpoints.get(i).getAddress();
Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
if (hostIndexQueue == null) {
hostIndexQueue = Lists.newLinkedList();
endpointHostIndexListMap.put(hostname, hostIndexQueue);
}
hostIndexQueue.add(i);
}
Set<Entry<HRegionInfo, ServerName>> regionsToAssignSet =
Sets.newHashSet(regionsToScan.entrySet());
/*
* First, we assign regions which are hosted on region servers running on drillbit endpoints
*/
for (Iterator<Entry<HRegionInfo, ServerName>> regionsIterator = regionsToAssignSet.iterator();
regionsIterator.hasNext(); /*nothing*/ ) {
Entry<HRegionInfo, ServerName> regionEntry = regionsIterator.next();
/*
* Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region
*/
Queue<Integer> endpointIndexlist =
endpointHostIndexListMap.get(regionEntry.getValue().getHostname());
if (endpointIndexlist != null) {
Integer slotIndex = endpointIndexlist.poll();
List<HBaseSubScanSpec> endpointSlotScanList = endpointFragmentMapping.get(slotIndex);
endpointSlotScanList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
// add to the tail of the slot list, to add more later in round robin fashion
endpointIndexlist.offer(slotIndex);
// this region has been assigned
regionsIterator.remove();
}
}
/*
* Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more.
*/
PriorityQueue<List<HBaseSubScanSpec>> minHeap =
new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
PriorityQueue<List<HBaseSubScanSpec>> maxHeap =
new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
for (List<HBaseSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
if (listOfScan.size() < minPerEndpointSlot) {
minHeap.offer(listOfScan);
} else if (listOfScan.size() > minPerEndpointSlot) {
maxHeap.offer(listOfScan);
}
}
/*
* Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments.
*/
if (regionsToAssignSet.size() > 0) {
for (Entry<HRegionInfo, ServerName> regionEntry : regionsToAssignSet) {
List<HBaseSubScanSpec> smallestList = minHeap.poll();
smallestList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
if (smallestList.size() < minPerEndpointSlot) {
minHeap.offer(smallestList);
}
}
}
/*
* While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more.
*/
while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
List<HBaseSubScanSpec> smallestList = minHeap.poll();
List<HBaseSubScanSpec> largestList = maxHeap.poll();
smallestList.add(largestList.remove(largestList.size() - 1));
if (largestList.size() > minPerEndpointSlot) {
maxHeap.offer(largestList);
}
if (smallestList.size() < minPerEndpointSlot) {
minHeap.offer(smallestList);
}
}
/* no slot should be empty at this point */
assert (minHeap.peek() == null || minHeap.peek().size() > 0)
: String.format(
"Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.",
incomingEndpoints,
endpointFragmentMapping.toString());
logger.debug(
"Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}",
watch.elapsed(TimeUnit.NANOSECONDS) / 1000,
incomingEndpoints,
endpointFragmentMapping.toString());
}
/**
* Runs a filter.
*
* @param pipelineConfigurationFile Pipeline configuration file.
* @param duration Timeout period.
* @param variables Substitution key-value pairs into pipeline configuration file.
* @throws IOException if configuration cannot be loaded.
*/
protected void benchmark(
final String pipelineConfigurationFile,
final Duration duration,
final ImmutableMap<String, String> variables)
throws IOException {
// Replace any variables in the configuration file
String configuration =
Resources.toString(Resources.getResource(pipelineConfigurationFile), Charsets.UTF_8);
for (final Map.Entry<String, String> entry : variables.entrySet()) {
configuration = configuration.replace(entry.getKey(), entry.getValue());
}
// Load the specified stock configuration
final PipelineConfiguration stockPipelineConfiguration =
new StaticConfiguration.Builder()
.addSource(new JsonNodeLiteralSource.Builder().setSource(configuration).build())
.setObjectMapper(PipelineConfiguration.createObjectMapper(_injector))
.build()
.getRequiredAs(PipelineConfiguration.class);
// Canary tracking
LOGGER.info(
String.format("Expected canaries; periods=%s", stockPipelineConfiguration.getPeriods()));
final CountDownLatch latch = new CountDownLatch(stockPipelineConfiguration.getPeriods().size());
final Set<Period> periods = Sets.newConcurrentHashSet();
// Create custom "canary" sink
final ListeningSink sink =
new ListeningSink(
(periodicData) -> {
if (periodicData != null) {
for (final String metricName : periodicData.getData().keys()) {
if (TestFileGenerator.CANARY.equals(metricName)) {
if (periods.add(periodicData.getPeriod())) {
LOGGER.info(
String.format(
"Canary flew; filter=%s, period=%s",
this.getClass(), periodicData.getPeriod()));
latch.countDown();
}
}
}
}
return null;
});
// Add the custom "canary" sink
final List<Sink> benchmarkSinks = Lists.newArrayList(stockPipelineConfiguration.getSinks());
benchmarkSinks.add(sink);
// Create the custom configuration
final PipelineConfiguration benchmarkPipelineConfiguration =
OvalBuilder.<PipelineConfiguration, PipelineConfiguration.Builder>clone(
stockPipelineConfiguration)
.setSinks(benchmarkSinks)
.build();
// Instantiate the pipeline
final Pipeline pipeline = new Pipeline(benchmarkPipelineConfiguration);
// Execute the pipeline until the canary flies the coop
try {
LOGGER.debug(
String.format("Launching pipeline; configuration=%s", pipelineConfigurationFile));
final Stopwatch timer = Stopwatch.createUnstarted();
timer.start();
pipeline.launch();
if (!latch.await(duration.getMillis(), TimeUnit.MILLISECONDS)) {
LOGGER.error("Test timed out");
throw new RuntimeException("Test timed out");
}
timer.stop();
LOGGER.info(
String.format(
"Performance filter result; filter=%s, seconds=%s",
this.getClass(), timer.elapsed(TimeUnit.SECONDS)));
} catch (final InterruptedException e) {
Thread.interrupted();
throw new RuntimeException("Test interrupted");
} finally {
pipeline.shutdown();
}
}
/** starts the timer, either initially or if {@link #pause()}d; no-op if already running */
public synchronized CountdownTimer start() {
if (!stopwatch.isRunning()) stopwatch.start();
return this;
}
@Test
@Ignore
public void testPerformance(
@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection)
throws Exception {
DrillConfig c = DrillConfig.create();
FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
FragmentContext context =
new FragmentContext(
bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
// new NonStrictExpectations() {
// {
// context.getAllocator(); result = BufferAllocator.getAllocator(DrillConfig.create());
// }
// };
final String fileName = "/tmp/parquet_test_performance.parquet";
HashMap<String, FieldInfo> fields = new HashMap<>();
ParquetTestProperties props =
new ParquetTestProperties(1, 20 * 1000 * 1000, DEFAULT_BYTES_PER_PAGE, fields);
populateFieldInfoMap(props);
// generateParquetFile(fileName, props);
Configuration dfsConfig = new Configuration();
List<Footer> footers = ParquetFileReader.readFooters(dfsConfig, new Path(fileName));
Footer f = footers.iterator().next();
List<SchemaPath> columns = Lists.newArrayList();
columns.add(new SchemaPath("_MAP.integer", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.bigInt", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.f", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.d", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.b", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN));
columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN));
int totalRowCount = 0;
FileSystem fs = new CachedSingleFileSystem(fileName);
BufferAllocator allocator = new TopLevelAllocator();
for (int i = 0; i < 25; i++) {
ParquetRecordReader rr =
new ParquetRecordReader(
context,
256000,
fileName,
0,
fs,
new CodecFactoryExposer(dfsConfig),
f.getParquetMetadata(),
columns);
TestOutputMutator mutator = new TestOutputMutator(allocator);
rr.setup(mutator);
Stopwatch watch = new Stopwatch();
watch.start();
int rowCount = 0;
while ((rowCount = rr.next()) > 0) {
totalRowCount += rowCount;
}
System.out.println(
String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
rr.cleanup();
}
allocator.close();
System.out.println(String.format("Total row count %s", totalRowCount));
}
@Override
public void queryIdArrived(QueryId queryId) {
w.start();
this.queryId = queryId;
}