@Override
public void record() {
super.record();
getStatistics("keySearch", count).addValue(keySearch.elapsedTime(TimeUnit.MICROSECONDS));
getStatistics("hashGet", count).addValue(hashGet.elapsedTime(TimeUnit.MICROSECONDS));
keySearch.reset();
hashGet.reset();
}
@Override
public List<EndpointAffinity> getOperatorAffinity() {
watch.reset();
watch.start();
Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
endpointMap.put(ep.getAddress(), ep);
}
Map<DrillbitEndpoint, EndpointAffinity> affinityMap =
new HashMap<DrillbitEndpoint, EndpointAffinity>();
for (ServerName sn : regionsToScan.values()) {
DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
if (ep != null) {
EndpointAffinity affinity = affinityMap.get(ep);
if (affinity == null) {
affinityMap.put(ep, new EndpointAffinity(ep, 1));
} else {
affinity.addAffinity(1);
}
}
}
logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
return Lists.newArrayList(affinityMap.values());
}
public void start() {
if (watch.isRunning()) {
watch.reset().start();
} else {
watch.start();
}
}
protected void createAndRunAServiceInGroup(String group) throws RunNodesException {
// note that some cloud providers do not support mixed case tag names
ImmutableMap<String, String> userMetadata = ImmutableMap.<String, String>of("test", group);
ImmutableSet<String> tags = ImmutableSet.of(group);
Stopwatch watch = Stopwatch.createStarted();
template = buildTemplate(client.templateBuilder());
template
.getOptions()
.inboundPorts(22, 8080)
.blockOnPort(22, 300)
.userMetadata(userMetadata)
.tags(tags);
NodeMetadata node = getOnlyElement(client.createNodesInGroup(group, 1, template));
long createSeconds = watch.elapsed(TimeUnit.SECONDS);
final String nodeId = node.getId();
checkUserMetadataContains(node, userMetadata);
checkTagsInNodeEquals(node, tags);
getAnonymousLogger()
.info(
format(
"<< available node(%s) os(%s) in %ss",
node.getId(), node.getOperatingSystem(), createSeconds));
watch.reset().start();
client.runScriptOnNode(nodeId, JettyStatements.install(), nameTask("configure-jetty"));
long configureSeconds = watch.elapsed(TimeUnit.SECONDS);
getAnonymousLogger()
.info(
format(
"<< configured node(%s) with %s and jetty %s in %ss",
nodeId,
exec(nodeId, "java -fullversion"),
exec(nodeId, JettyStatements.version()),
configureSeconds));
trackAvailabilityOfProcessOnNode(JettyStatements.start(), "start jetty", node);
client.runScriptOnNode(
nodeId, JettyStatements.stop(), runAsRoot(false).wrapInInitScript(false));
trackAvailabilityOfProcessOnNode(JettyStatements.start(), "start jetty", node);
}
private void test(String description, int iterations, Runnable task) {
LOGGER.info("Running test: " + description);
long best = Long.MAX_VALUE;
Stopwatch stopwatch = new Stopwatch();
for (int i = 0; i < iterations; i++) {
stopwatch.start();
task.run();
stopwatch.stop();
long elapsed = stopwatch.elapsed(TimeUnit.MICROSECONDS);
best = Math.min(best, elapsed);
stopwatch.reset();
}
LOGGER.info("Finished test " + description + " in " + best + "µs");
}
private void narrowByRule(ConstrainedTerm constrainedTerm, Rule rule) {
stopwatch.reset();
stopwatch.start();
constrainedTermResults = new ArrayList<ConstrainedTerm>();
SymbolicConstraint leftHandSideConstraint =
new SymbolicConstraint(constrainedTerm.termContext());
leftHandSideConstraint.addAll(rule.requires());
for (Variable variable : rule.freshVariables()) {
leftHandSideConstraint.add(variable, IntToken.fresh());
}
ConstrainedTerm leftHandSide =
new ConstrainedTerm(
rule.leftHandSide(),
rule.lookups().getSymbolicConstraint(constrainedTerm.termContext()),
leftHandSideConstraint,
constrainedTerm.termContext());
for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
constraint.addAll(rule.ensures());
/* rename rule variables in the constraints */
Map<Variable, Variable> freshSubstitution = constraint.rename(rule.variableSet());
Term result = rule.rightHandSide();
/* rename rule variables in the rule RHS */
result = result.substituteWithBinders(freshSubstitution, constrainedTerm.termContext());
/* apply the constraints substitution on the rule RHS */
result =
result.substituteWithBinders(constraint.substitution(), constrainedTerm.termContext());
/* evaluate pending functions in the rule RHS */
result = result.evaluate(constrainedTerm.termContext());
/* eliminate anonymous variables */
constraint.eliminateAnonymousVariables();
/* compute all results */
constrainedTermResults.add(
new ConstrainedTerm(result, constraint, constrainedTerm.termContext()));
}
stopwatch.stop();
}
// apply rule by matching
private void rewriteByRule(Term term, Rule rule) {
stopwatch.reset();
stopwatch.start();
termResults = new ArrayList<Term>();
TermContext context = TermContext.of(definition);
ConstrainedTerm constrainedTerm = new ConstrainedTerm(term, context);
SymbolicConstraint leftHandSideConstraint = new SymbolicConstraint(context);
leftHandSideConstraint.addAll(rule.requires());
for (Variable variable : rule.freshVariables()) {
leftHandSideConstraint.add(variable, IntToken.fresh());
}
ConstrainedTerm leftHandSide =
new ConstrainedTerm(
rule.leftHandSide(),
rule.lookups().getSymbolicConstraint(context),
leftHandSideConstraint,
context);
for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
if (!constraint.isMatching(leftHandSide)) {
continue;
}
constraint.orientSubstitution(leftHandSide.variableSet());
Term result = rule.rightHandSide();
/* apply the constraints substitution on the rule RHS */
result = result.substituteAndEvaluate(constraint.substitution(), context);
/* compute all results */
termResults.add(result);
}
stopwatch.stop();
}
/**
* Code for each 'client' to run.
*
* @param id
* @param c
* @param sharedConnection
* @throws IOException
*/
static void cycle(int id, final Configuration c, final HConnection sharedConnection)
throws IOException {
HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE);
table.setAutoFlushTo(false);
long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000);
long startTime = System.currentTimeMillis();
final int printInterval = 100000;
Random rd = new Random(id);
boolean get = c.getBoolean("hbase.test.do.gets", false);
try {
Stopwatch stopWatch = new Stopwatch();
stopWatch.start();
for (int i = 0; i < namespaceSpan; i++) {
byte[] b = format(rd.nextLong());
if (get) {
Get g = new Get(b);
table.get(g);
} else {
Put p = new Put(b);
p.add(HConstants.CATALOG_FAMILY, b, b);
table.put(p);
}
if (i % printInterval == 0) {
LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis());
stopWatch.reset();
stopWatch.start();
}
}
LOG.info(
"Finished a cycle putting "
+ namespaceSpan
+ " in "
+ (System.currentTimeMillis() - startTime)
+ "ms");
} finally {
table.close();
}
}
protected ServiceStats trackAvailabilityOfProcessOnNode(
Statement process, String processName, NodeMetadata node) {
ServiceStats stats = new ServiceStats();
Stopwatch watch = Stopwatch.createStarted();
ExecResponse exec =
client.runScriptOnNode(node.getId(), process, runAsRoot(false).wrapInInitScript(false));
stats.backgroundProcessMilliseconds = watch.elapsed(TimeUnit.MILLISECONDS);
watch.reset().start();
HostAndPort socket = null;
try {
socket = openSocketFinder.findOpenSocketOnNode(node, 8080, 600, TimeUnit.SECONDS);
} catch (NoSuchElementException e) {
throw new NoSuchElementException(
format("%s%n%s%s", e.getMessage(), exec.getOutput(), exec.getError()));
}
stats.socketOpenMilliseconds = watch.elapsed(TimeUnit.MILLISECONDS);
getAnonymousLogger()
.info(format("<< %s on node(%s)[%s] %s", processName, node.getId(), socket, stats));
return stats;
}
private synchronized void resetErrors() {
errorStopwatch.reset();
}
/** @param incomingEndpoints */
@Override
public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
watch.reset();
watch.start();
final int numSlots = incomingEndpoints.size();
Preconditions.checkArgument(
numSlots <= regionsToScan.size(),
String.format(
"Incoming endpoints %d is greater than number of scan regions %d",
numSlots, regionsToScan.size()));
/*
* Minimum/Maximum number of assignment per slot
*/
final int minPerEndpointSlot = (int) Math.floor((double) regionsToScan.size() / numSlots);
final int maxPerEndpointSlot = (int) Math.ceil((double) regionsToScan.size() / numSlots);
/*
* initialize (endpoint index => HBaseSubScanSpec list) map
*/
endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);
/*
* another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list
*/
Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();
/*
* Initialize these two maps
*/
for (int i = 0; i < numSlots; ++i) {
endpointFragmentMapping.put(i, new ArrayList<HBaseSubScanSpec>(maxPerEndpointSlot));
String hostname = incomingEndpoints.get(i).getAddress();
Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
if (hostIndexQueue == null) {
hostIndexQueue = Lists.newLinkedList();
endpointHostIndexListMap.put(hostname, hostIndexQueue);
}
hostIndexQueue.add(i);
}
Set<Entry<HRegionInfo, ServerName>> regionsToAssignSet =
Sets.newHashSet(regionsToScan.entrySet());
/*
* First, we assign regions which are hosted on region servers running on drillbit endpoints
*/
for (Iterator<Entry<HRegionInfo, ServerName>> regionsIterator = regionsToAssignSet.iterator();
regionsIterator.hasNext(); /*nothing*/ ) {
Entry<HRegionInfo, ServerName> regionEntry = regionsIterator.next();
/*
* Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region
*/
Queue<Integer> endpointIndexlist =
endpointHostIndexListMap.get(regionEntry.getValue().getHostname());
if (endpointIndexlist != null) {
Integer slotIndex = endpointIndexlist.poll();
List<HBaseSubScanSpec> endpointSlotScanList = endpointFragmentMapping.get(slotIndex);
endpointSlotScanList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
// add to the tail of the slot list, to add more later in round robin fashion
endpointIndexlist.offer(slotIndex);
// this region has been assigned
regionsIterator.remove();
}
}
/*
* Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more.
*/
PriorityQueue<List<HBaseSubScanSpec>> minHeap =
new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
PriorityQueue<List<HBaseSubScanSpec>> maxHeap =
new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
for (List<HBaseSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
if (listOfScan.size() < minPerEndpointSlot) {
minHeap.offer(listOfScan);
} else if (listOfScan.size() > minPerEndpointSlot) {
maxHeap.offer(listOfScan);
}
}
/*
* Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments.
*/
if (regionsToAssignSet.size() > 0) {
for (Entry<HRegionInfo, ServerName> regionEntry : regionsToAssignSet) {
List<HBaseSubScanSpec> smallestList = minHeap.poll();
smallestList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
if (smallestList.size() < minPerEndpointSlot) {
minHeap.offer(smallestList);
}
}
}
/*
* While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more.
*/
while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
List<HBaseSubScanSpec> smallestList = minHeap.poll();
List<HBaseSubScanSpec> largestList = maxHeap.poll();
smallestList.add(largestList.remove(largestList.size() - 1));
if (largestList.size() > minPerEndpointSlot) {
maxHeap.offer(largestList);
}
if (smallestList.size() < minPerEndpointSlot) {
minHeap.offer(smallestList);
}
}
/* no slot should be empty at this point */
assert (minHeap.peek() == null || minHeap.peek().size() > 0)
: String.format(
"Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.",
incomingEndpoints,
endpointFragmentMapping.toString());
logger.debug(
"Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}",
watch.elapsed(TimeUnit.NANOSECONDS) / 1000,
incomingEndpoints,
endpointFragmentMapping.toString());
}
@Override
public void messageArrived(String topic, MqttMessage message) {
try {
Stopwatch sw = Stopwatch.createStarted();
String msg = new String(message.getPayload());
JsonParser parser = new JsonParser();
JsonObject object = parser.parse(msg).getAsJsonObject();
// XXX: workaround for demo!!!!
String sensorId = topic;
String[] segments = topic.split("/");
if (segments.length > 2) {
sensorId = segments[1] + segments[2];
}
long elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
if (elapsed > 5) {
System.out.println("Phase parse: " + elapsed);
}
sw.reset();
// Use IncQuery Base Indexer to find the sensor
Sensor selectedSensor =
(Sensor) navigationHelper.findByAttributeValue(sensorId).iterator().next().getEObject();
elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
if (elapsed > 10) {
System.out.println("Phase indexer: " + elapsed);
}
Payload sensorPayload = selectedSensor.getLastReceivedPayload();
NavigationHelper paramNavHelper =
IncQueryBaseFactory.getInstance().createNavigationHelper(sensorPayload, true, null);
// Get parameters from model using the JSON file content
for (Entry<String, JsonElement> parameter : object.entrySet()) {
// Use IncQuery Base Indexer to find parameter
Set<Setting> settings = paramNavHelper.findByAttributeValue(parameter.getKey());
if (settings.isEmpty()) continue;
DataParameter paramValue = (DataParameter) settings.iterator().next().getEObject();
// Get the parameter new value from the message
JsonElement newValue = parameter.getValue();
// Find parameter type, and set the new value
if (paramValue.getType().equals("int")) {
((IntParameter) paramValue).setValue(newValue.getAsInt());
} else if (paramValue.getType().equals("double")) {
((DoubleParameter) paramValue).setValue(newValue.getAsDouble());
} else if (paramValue.getType().equals("string")) {
((StringParameter) paramValue).setValue(newValue.getAsString());
} else if (paramValue.getType().equals("boolean")) {
((BooleanParameter) paramValue).setValue(newValue.getAsBoolean());
} else if (paramValue.getType().equals("long")) {
((LongParameter) paramValue).setValue(newValue.getAsLong());
}
}
elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
if (elapsed > 10) {
System.out.println("Phase update: " + elapsed);
}
} catch (Exception e) {
LOGGER.error(e.getMessage());
}
}
/**
* Dame todos los paises
*
* <p>http://api.geonames.org/countryInfo?lang=es&username=gsantosgo&style=full
* http://api.geonames.org/search?featureCode=PCLI&featureCode=PCL&lang=es&username=gsantosgo&style=full
*
* @param args
*/
public static void main(String args[]) {
MyWebService.setUserName("gsantosgo");
System.out.println("======================================");
System.out.println(" Geonames Server: " + MyWebService.getGeoNamesServer());
System.out.println(" Geonames Server: " + MyWebService.getUserName());
/*
// Configuracion PROXY. Para conectarse correctamente este necesario configurar el PROXY.
System.setProperty("http.proxySet","true");
System.setProperty("http.proxyHost","10.14.79.204");
System.setProperty("http.proxyPort","8080");
System.setProperty("http.proxyUser","");
System.setProperty("http.proxyPassword","");
*/
Stopwatch stopwatch = new Stopwatch();
// Step 1. CountryInfo ======
stopwatch.start();
ToponymSearchCriteria toponymSearchCriteria = new ToponymSearchCriteria();
toponymSearchCriteria.setLanguage("es");
toponymSearchCriteria.setStyle(Style.FULL);
ToponymSearchResult toponymSearchResult = new ToponymSearchResult();
try {
toponymSearchResult = MyWebService.countryInfo(toponymSearchCriteria);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Número de Paises: " + toponymSearchResult.getTotalResultsCount());
List<Toponym> toponymList = toponymSearchResult.getToponyms();
// for (Toponym toponym : toponymList) {
// System.out.println(toponym);
// }
stopwatch.stop();
System.out.println(
String.format(
"Tiempo transcurrido en %d miliseconds: ",
stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));
stopwatch.reset();
stopwatch.start();
// Step 2. Country Search
ToponymSearchCriteria toponymCountrySearchCriteria = new ToponymSearchCriteria();
toponymCountrySearchCriteria.setLanguage("es");
toponymCountrySearchCriteria.setStyle(Style.FULL);
toponymCountrySearchCriteria.setFeatureClass(FeatureClass.A);
toponymCountrySearchCriteria.setFeatureCodes(
new String[] {"PCL", "PCLD", "PCLF", "PCLI", "PCLIX", "PCLS"});
ToponymSearchResult toponymCountrySearchResult = new ToponymSearchResult();
try {
toponymCountrySearchResult = MyWebService.search(toponymCountrySearchCriteria);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Número de Paises: " + toponymCountrySearchResult.getTotalResultsCount());
List<Toponym> toponymCountryList = toponymCountrySearchResult.getToponyms();
Map<Integer, Toponym> mapToponyms = Maps.newHashMap();
for (Toponym toponym : toponymCountryList) {
mapToponyms.put(toponym.getGeoNameId(), toponym);
}
int count = 0;
for (Toponym toponym : toponymList) {
Toponym mapToponym = mapToponyms.get(toponym.getGeoNameId());
if (mapToponym == null) {
count++;
}
}
System.out.println(" ==>" + count);
stopwatch.stop();
System.out.println(
String.format(
"Tiempo transcurrido en %d miliseconds: ",
stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));
}
public void touch() {
stopwatch.reset();
}
@Test(dependsOnGroups = "example.ipc.server")
public void clientTest() throws IOException, ServiceException, InterruptedException {
PeerInfo client = new PeerInfo(socketAddress.getHostName(), 1234);
ThreadPoolCallExecutor executor = new ThreadPoolCallExecutor(3, 10);
DuplexTcpClientBootstrap bootstrap =
new DuplexTcpClientBootstrap(
client,
new NioClientSocketChannelFactory(
Executors.newCachedThreadPool(), Executors.newCachedThreadPool()),
executor);
bootstrap.setOption("connectTimeoutMillis", 10000);
bootstrap.setOption("connectResponseTimeoutMillis", 10000);
bootstrap.setOption("receiveBufferSize", 1048576);
bootstrap.setOption("tcpNoDelay", false);
RpcClientChannel channel = bootstrap.peerWith(socketAddress);
// blocking calll
DataService.BlockingInterface dataService = DataService.newBlockingStub(channel);
RpcController controller = channel.newRpcController();
// make request
GetRequest request = GetRequest.newBuilder().setRow(ByteString.copyFromUtf8("row1")).build();
final Stopwatch stopwatch = new Stopwatch().start();
GetResponse response = dataService.getData(controller, request);
stopwatch.stop();
System.out.println(response.getDataList());
System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());
// do it again since the socket is open
stopwatch.reset().start();
response = dataService.getData(controller, request);
stopwatch.stop();
System.out.println(response.getDataList());
System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());
// non-blocking
DataService.Stub stub = DataService.newStub(channel);
final Object lock = new Object();
stopwatch.reset().start();
stub.getData(
controller,
request,
new RpcCallback<GetResponse>() {
public void run(final GetResponse parameter) {
System.out.println("Non-Blocking Callback");
System.out.println(parameter.getDataList());
stopwatch.stop();
System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());
synchronized (lock) {
lock.notify();
}
}
});
synchronized (lock) {
lock.wait();
}
}