@Override
  public void record() {
    super.record();

    getStatistics("keySearch", count).addValue(keySearch.elapsedTime(TimeUnit.MICROSECONDS));
    getStatistics("hashGet", count).addValue(hashGet.elapsedTime(TimeUnit.MICROSECONDS));

    keySearch.reset();
    hashGet.reset();
  }
  @Override
  public List<EndpointAffinity> getOperatorAffinity() {
    watch.reset();
    watch.start();
    Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
    for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
      endpointMap.put(ep.getAddress(), ep);
    }

    Map<DrillbitEndpoint, EndpointAffinity> affinityMap =
        new HashMap<DrillbitEndpoint, EndpointAffinity>();
    for (ServerName sn : regionsToScan.values()) {
      DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
      if (ep != null) {
        EndpointAffinity affinity = affinityMap.get(ep);
        if (affinity == null) {
          affinityMap.put(ep, new EndpointAffinity(ep, 1));
        } else {
          affinity.addAffinity(1);
        }
      }
    }
    logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
    return Lists.newArrayList(affinityMap.values());
  }
Example #3
0
 public void start() {
   if (watch.isRunning()) {
     watch.reset().start();
   } else {
     watch.start();
   }
 }
  protected void createAndRunAServiceInGroup(String group) throws RunNodesException {
    // note that some cloud providers do not support mixed case tag names
    ImmutableMap<String, String> userMetadata = ImmutableMap.<String, String>of("test", group);

    ImmutableSet<String> tags = ImmutableSet.of(group);
    Stopwatch watch = Stopwatch.createStarted();

    template = buildTemplate(client.templateBuilder());
    template
        .getOptions()
        .inboundPorts(22, 8080)
        .blockOnPort(22, 300)
        .userMetadata(userMetadata)
        .tags(tags);

    NodeMetadata node = getOnlyElement(client.createNodesInGroup(group, 1, template));
    long createSeconds = watch.elapsed(TimeUnit.SECONDS);

    final String nodeId = node.getId();

    checkUserMetadataContains(node, userMetadata);
    checkTagsInNodeEquals(node, tags);

    getAnonymousLogger()
        .info(
            format(
                "<< available node(%s) os(%s) in %ss",
                node.getId(), node.getOperatingSystem(), createSeconds));

    watch.reset().start();

    client.runScriptOnNode(nodeId, JettyStatements.install(), nameTask("configure-jetty"));

    long configureSeconds = watch.elapsed(TimeUnit.SECONDS);

    getAnonymousLogger()
        .info(
            format(
                "<< configured node(%s) with %s and jetty %s in %ss",
                nodeId,
                exec(nodeId, "java -fullversion"),
                exec(nodeId, JettyStatements.version()),
                configureSeconds));

    trackAvailabilityOfProcessOnNode(JettyStatements.start(), "start jetty", node);

    client.runScriptOnNode(
        nodeId, JettyStatements.stop(), runAsRoot(false).wrapInInitScript(false));

    trackAvailabilityOfProcessOnNode(JettyStatements.start(), "start jetty", node);
  }
Example #5
0
  private void test(String description, int iterations, Runnable task) {
    LOGGER.info("Running test: " + description);

    long best = Long.MAX_VALUE;
    Stopwatch stopwatch = new Stopwatch();

    for (int i = 0; i < iterations; i++) {
      stopwatch.start();
      task.run();
      stopwatch.stop();
      long elapsed = stopwatch.elapsed(TimeUnit.MICROSECONDS);
      best = Math.min(best, elapsed);
      stopwatch.reset();
    }

    LOGGER.info("Finished test " + description + " in " + best + "µs");
  }
Example #6
0
  private void narrowByRule(ConstrainedTerm constrainedTerm, Rule rule) {
    stopwatch.reset();
    stopwatch.start();

    constrainedTermResults = new ArrayList<ConstrainedTerm>();

    SymbolicConstraint leftHandSideConstraint =
        new SymbolicConstraint(constrainedTerm.termContext());
    leftHandSideConstraint.addAll(rule.requires());
    for (Variable variable : rule.freshVariables()) {
      leftHandSideConstraint.add(variable, IntToken.fresh());
    }

    ConstrainedTerm leftHandSide =
        new ConstrainedTerm(
            rule.leftHandSide(),
            rule.lookups().getSymbolicConstraint(constrainedTerm.termContext()),
            leftHandSideConstraint,
            constrainedTerm.termContext());

    for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
      constraint.addAll(rule.ensures());
      /* rename rule variables in the constraints */
      Map<Variable, Variable> freshSubstitution = constraint.rename(rule.variableSet());

      Term result = rule.rightHandSide();
      /* rename rule variables in the rule RHS */
      result = result.substituteWithBinders(freshSubstitution, constrainedTerm.termContext());
      /* apply the constraints substitution on the rule RHS */
      result =
          result.substituteWithBinders(constraint.substitution(), constrainedTerm.termContext());
      /* evaluate pending functions in the rule RHS */
      result = result.evaluate(constrainedTerm.termContext());
      /* eliminate anonymous variables */
      constraint.eliminateAnonymousVariables();

      /* compute all results */
      constrainedTermResults.add(
          new ConstrainedTerm(result, constraint, constrainedTerm.termContext()));
    }

    stopwatch.stop();
  }
Example #7
0
  // apply rule by matching
  private void rewriteByRule(Term term, Rule rule) {
    stopwatch.reset();
    stopwatch.start();

    termResults = new ArrayList<Term>();

    TermContext context = TermContext.of(definition);
    ConstrainedTerm constrainedTerm = new ConstrainedTerm(term, context);

    SymbolicConstraint leftHandSideConstraint = new SymbolicConstraint(context);
    leftHandSideConstraint.addAll(rule.requires());
    for (Variable variable : rule.freshVariables()) {
      leftHandSideConstraint.add(variable, IntToken.fresh());
    }

    ConstrainedTerm leftHandSide =
        new ConstrainedTerm(
            rule.leftHandSide(),
            rule.lookups().getSymbolicConstraint(context),
            leftHandSideConstraint,
            context);

    for (SymbolicConstraint constraint : constrainedTerm.unify(leftHandSide)) {
      if (!constraint.isMatching(leftHandSide)) {
        continue;
      }

      constraint.orientSubstitution(leftHandSide.variableSet());

      Term result = rule.rightHandSide();
      /* apply the constraints substitution on the rule RHS */
      result = result.substituteAndEvaluate(constraint.substitution(), context);

      /* compute all results */
      termResults.add(result);
    }

    stopwatch.stop();
  }
 /**
  * Code for each 'client' to run.
  *
  * @param id
  * @param c
  * @param sharedConnection
  * @throws IOException
  */
 static void cycle(int id, final Configuration c, final HConnection sharedConnection)
     throws IOException {
   HTableInterface table = sharedConnection.getTable(BIG_USER_TABLE);
   table.setAutoFlushTo(false);
   long namespaceSpan = c.getLong("hbase.test.namespace.span", 1000000);
   long startTime = System.currentTimeMillis();
   final int printInterval = 100000;
   Random rd = new Random(id);
   boolean get = c.getBoolean("hbase.test.do.gets", false);
   try {
     Stopwatch stopWatch = new Stopwatch();
     stopWatch.start();
     for (int i = 0; i < namespaceSpan; i++) {
       byte[] b = format(rd.nextLong());
       if (get) {
         Get g = new Get(b);
         table.get(g);
       } else {
         Put p = new Put(b);
         p.add(HConstants.CATALOG_FAMILY, b, b);
         table.put(p);
       }
       if (i % printInterval == 0) {
         LOG.info("Put " + printInterval + "/" + stopWatch.elapsedMillis());
         stopWatch.reset();
         stopWatch.start();
       }
     }
     LOG.info(
         "Finished a cycle putting "
             + namespaceSpan
             + " in "
             + (System.currentTimeMillis() - startTime)
             + "ms");
   } finally {
     table.close();
   }
 }
  protected ServiceStats trackAvailabilityOfProcessOnNode(
      Statement process, String processName, NodeMetadata node) {
    ServiceStats stats = new ServiceStats();
    Stopwatch watch = Stopwatch.createStarted();
    ExecResponse exec =
        client.runScriptOnNode(node.getId(), process, runAsRoot(false).wrapInInitScript(false));
    stats.backgroundProcessMilliseconds = watch.elapsed(TimeUnit.MILLISECONDS);
    watch.reset().start();

    HostAndPort socket = null;
    try {
      socket = openSocketFinder.findOpenSocketOnNode(node, 8080, 600, TimeUnit.SECONDS);
    } catch (NoSuchElementException e) {
      throw new NoSuchElementException(
          format("%s%n%s%s", e.getMessage(), exec.getOutput(), exec.getError()));
    }

    stats.socketOpenMilliseconds = watch.elapsed(TimeUnit.MILLISECONDS);

    getAnonymousLogger()
        .info(format("<< %s on node(%s)[%s] %s", processName, node.getId(), socket, stats));
    return stats;
  }
Example #10
0
 private synchronized void resetErrors() {
   errorStopwatch.reset();
 }
  /** @param incomingEndpoints */
  @Override
  public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
    watch.reset();
    watch.start();

    final int numSlots = incomingEndpoints.size();
    Preconditions.checkArgument(
        numSlots <= regionsToScan.size(),
        String.format(
            "Incoming endpoints %d is greater than number of scan regions %d",
            numSlots, regionsToScan.size()));

    /*
     * Minimum/Maximum number of assignment per slot
     */
    final int minPerEndpointSlot = (int) Math.floor((double) regionsToScan.size() / numSlots);
    final int maxPerEndpointSlot = (int) Math.ceil((double) regionsToScan.size() / numSlots);

    /*
     * initialize (endpoint index => HBaseSubScanSpec list) map
     */
    endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);

    /*
     * another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list
     */
    Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();

    /*
     * Initialize these two maps
     */
    for (int i = 0; i < numSlots; ++i) {
      endpointFragmentMapping.put(i, new ArrayList<HBaseSubScanSpec>(maxPerEndpointSlot));
      String hostname = incomingEndpoints.get(i).getAddress();
      Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
      if (hostIndexQueue == null) {
        hostIndexQueue = Lists.newLinkedList();
        endpointHostIndexListMap.put(hostname, hostIndexQueue);
      }
      hostIndexQueue.add(i);
    }

    Set<Entry<HRegionInfo, ServerName>> regionsToAssignSet =
        Sets.newHashSet(regionsToScan.entrySet());

    /*
     * First, we assign regions which are hosted on region servers running on drillbit endpoints
     */
    for (Iterator<Entry<HRegionInfo, ServerName>> regionsIterator = regionsToAssignSet.iterator();
        regionsIterator.hasNext(); /*nothing*/ ) {
      Entry<HRegionInfo, ServerName> regionEntry = regionsIterator.next();
      /*
       * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region
       */
      Queue<Integer> endpointIndexlist =
          endpointHostIndexListMap.get(regionEntry.getValue().getHostname());
      if (endpointIndexlist != null) {
        Integer slotIndex = endpointIndexlist.poll();
        List<HBaseSubScanSpec> endpointSlotScanList = endpointFragmentMapping.get(slotIndex);
        endpointSlotScanList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
        // add to the tail of the slot list, to add more later in round robin fashion
        endpointIndexlist.offer(slotIndex);
        // this region has been assigned
        regionsIterator.remove();
      }
    }

    /*
     * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more.
     */
    PriorityQueue<List<HBaseSubScanSpec>> minHeap =
        new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
    PriorityQueue<List<HBaseSubScanSpec>> maxHeap =
        new PriorityQueue<List<HBaseSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
    for (List<HBaseSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
      if (listOfScan.size() < minPerEndpointSlot) {
        minHeap.offer(listOfScan);
      } else if (listOfScan.size() > minPerEndpointSlot) {
        maxHeap.offer(listOfScan);
      }
    }

    /*
     * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments.
     */
    if (regionsToAssignSet.size() > 0) {
      for (Entry<HRegionInfo, ServerName> regionEntry : regionsToAssignSet) {
        List<HBaseSubScanSpec> smallestList = minHeap.poll();
        smallestList.add(regionInfoToSubScanSpec(regionEntry.getKey()));
        if (smallestList.size() < minPerEndpointSlot) {
          minHeap.offer(smallestList);
        }
      }
    }

    /*
     * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more.
     */
    while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
      List<HBaseSubScanSpec> smallestList = minHeap.poll();
      List<HBaseSubScanSpec> largestList = maxHeap.poll();
      smallestList.add(largestList.remove(largestList.size() - 1));
      if (largestList.size() > minPerEndpointSlot) {
        maxHeap.offer(largestList);
      }
      if (smallestList.size() < minPerEndpointSlot) {
        minHeap.offer(smallestList);
      }
    }

    /* no slot should be empty at this point */
    assert (minHeap.peek() == null || minHeap.peek().size() > 0)
        : String.format(
            "Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.",
            incomingEndpoints,
            endpointFragmentMapping.toString());

    logger.debug(
        "Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}",
        watch.elapsed(TimeUnit.NANOSECONDS) / 1000,
        incomingEndpoints,
        endpointFragmentMapping.toString());
  }
Example #12
0
  @Override
  public void messageArrived(String topic, MqttMessage message) {
    try {

      Stopwatch sw = Stopwatch.createStarted();

      String msg = new String(message.getPayload());
      JsonParser parser = new JsonParser();
      JsonObject object = parser.parse(msg).getAsJsonObject();

      // XXX: workaround for demo!!!!
      String sensorId = topic;
      String[] segments = topic.split("/");
      if (segments.length > 2) {
        sensorId = segments[1] + segments[2];
      }

      long elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
      if (elapsed > 5) {
        System.out.println("Phase parse: " + elapsed);
      }
      sw.reset();

      // Use IncQuery Base Indexer to find the sensor
      Sensor selectedSensor =
          (Sensor) navigationHelper.findByAttributeValue(sensorId).iterator().next().getEObject();

      elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
      if (elapsed > 10) {
        System.out.println("Phase indexer: " + elapsed);
      }

      Payload sensorPayload = selectedSensor.getLastReceivedPayload();

      NavigationHelper paramNavHelper =
          IncQueryBaseFactory.getInstance().createNavigationHelper(sensorPayload, true, null);
      // Get parameters from model using the JSON file content
      for (Entry<String, JsonElement> parameter : object.entrySet()) {
        // Use IncQuery Base Indexer to find parameter
        Set<Setting> settings = paramNavHelper.findByAttributeValue(parameter.getKey());
        if (settings.isEmpty()) continue;

        DataParameter paramValue = (DataParameter) settings.iterator().next().getEObject();
        // Get the parameter new value from the message
        JsonElement newValue = parameter.getValue();
        // Find parameter type, and set the new value
        if (paramValue.getType().equals("int")) {
          ((IntParameter) paramValue).setValue(newValue.getAsInt());
        } else if (paramValue.getType().equals("double")) {
          ((DoubleParameter) paramValue).setValue(newValue.getAsDouble());
        } else if (paramValue.getType().equals("string")) {
          ((StringParameter) paramValue).setValue(newValue.getAsString());
        } else if (paramValue.getType().equals("boolean")) {
          ((BooleanParameter) paramValue).setValue(newValue.getAsBoolean());
        } else if (paramValue.getType().equals("long")) {
          ((LongParameter) paramValue).setValue(newValue.getAsLong());
        }
      }

      elapsed = sw.elapsed(TimeUnit.MILLISECONDS);
      if (elapsed > 10) {
        System.out.println("Phase update: " + elapsed);
      }

    } catch (Exception e) {
      LOGGER.error(e.getMessage());
    }
  }
  /**
   * Dame todos los paises
   *
   * <p>http://api.geonames.org/countryInfo?lang=es&username=gsantosgo&style=full
   * http://api.geonames.org/search?featureCode=PCLI&featureCode=PCL&lang=es&username=gsantosgo&style=full
   *
   * @param args
   */
  public static void main(String args[]) {
    MyWebService.setUserName("gsantosgo");

    System.out.println("======================================");
    System.out.println(" Geonames Server: " + MyWebService.getGeoNamesServer());
    System.out.println(" Geonames Server: " + MyWebService.getUserName());

    /*
    // Configuracion PROXY. Para conectarse correctamente este necesario configurar el PROXY.
          System.setProperty("http.proxySet","true");
          System.setProperty("http.proxyHost","10.14.79.204");
          System.setProperty("http.proxyPort","8080");
          System.setProperty("http.proxyUser","");
          System.setProperty("http.proxyPassword","");
     */

    Stopwatch stopwatch = new Stopwatch();

    // Step 1. CountryInfo ======
    stopwatch.start();
    ToponymSearchCriteria toponymSearchCriteria = new ToponymSearchCriteria();
    toponymSearchCriteria.setLanguage("es");
    toponymSearchCriteria.setStyle(Style.FULL);

    ToponymSearchResult toponymSearchResult = new ToponymSearchResult();
    try {
      toponymSearchResult = MyWebService.countryInfo(toponymSearchCriteria);
    } catch (Exception e) {
      e.printStackTrace();
    }
    System.out.println("Número de Paises: " + toponymSearchResult.getTotalResultsCount());
    List<Toponym> toponymList = toponymSearchResult.getToponyms();
    // for (Toponym toponym : toponymList) {
    // System.out.println(toponym);
    //	}
    stopwatch.stop();
    System.out.println(
        String.format(
            "Tiempo transcurrido en %d miliseconds: ",
            stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));

    stopwatch.reset();
    stopwatch.start();

    // Step 2. Country Search
    ToponymSearchCriteria toponymCountrySearchCriteria = new ToponymSearchCriteria();
    toponymCountrySearchCriteria.setLanguage("es");
    toponymCountrySearchCriteria.setStyle(Style.FULL);
    toponymCountrySearchCriteria.setFeatureClass(FeatureClass.A);
    toponymCountrySearchCriteria.setFeatureCodes(
        new String[] {"PCL", "PCLD", "PCLF", "PCLI", "PCLIX", "PCLS"});

    ToponymSearchResult toponymCountrySearchResult = new ToponymSearchResult();
    try {
      toponymCountrySearchResult = MyWebService.search(toponymCountrySearchCriteria);
    } catch (Exception e) {
      e.printStackTrace();
    }
    System.out.println("Número de Paises: " + toponymCountrySearchResult.getTotalResultsCount());
    List<Toponym> toponymCountryList = toponymCountrySearchResult.getToponyms();
    Map<Integer, Toponym> mapToponyms = Maps.newHashMap();
    for (Toponym toponym : toponymCountryList) {
      mapToponyms.put(toponym.getGeoNameId(), toponym);
    }

    int count = 0;
    for (Toponym toponym : toponymList) {
      Toponym mapToponym = mapToponyms.get(toponym.getGeoNameId());
      if (mapToponym == null) {
        count++;
      }
    }
    System.out.println(" ==>" + count);

    stopwatch.stop();
    System.out.println(
        String.format(
            "Tiempo transcurrido en %d miliseconds: ",
            stopwatch.elapsedTime(TimeUnit.MILLISECONDS)));
  }
 public void touch() {
   stopwatch.reset();
 }
Example #15
0
  @Test(dependsOnGroups = "example.ipc.server")
  public void clientTest() throws IOException, ServiceException, InterruptedException {
    PeerInfo client = new PeerInfo(socketAddress.getHostName(), 1234);

    ThreadPoolCallExecutor executor = new ThreadPoolCallExecutor(3, 10);

    DuplexTcpClientBootstrap bootstrap =
        new DuplexTcpClientBootstrap(
            client,
            new NioClientSocketChannelFactory(
                Executors.newCachedThreadPool(), Executors.newCachedThreadPool()),
            executor);

    bootstrap.setOption("connectTimeoutMillis", 10000);
    bootstrap.setOption("connectResponseTimeoutMillis", 10000);
    bootstrap.setOption("receiveBufferSize", 1048576);
    bootstrap.setOption("tcpNoDelay", false);

    RpcClientChannel channel = bootstrap.peerWith(socketAddress);

    // blocking calll
    DataService.BlockingInterface dataService = DataService.newBlockingStub(channel);
    RpcController controller = channel.newRpcController();

    // make request
    GetRequest request = GetRequest.newBuilder().setRow(ByteString.copyFromUtf8("row1")).build();
    final Stopwatch stopwatch = new Stopwatch().start();
    GetResponse response = dataService.getData(controller, request);
    stopwatch.stop();
    System.out.println(response.getDataList());
    System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());

    // do it again since the socket is open
    stopwatch.reset().start();
    response = dataService.getData(controller, request);
    stopwatch.stop();
    System.out.println(response.getDataList());
    System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());

    // non-blocking
    DataService.Stub stub = DataService.newStub(channel);
    final Object lock = new Object();
    stopwatch.reset().start();
    stub.getData(
        controller,
        request,
        new RpcCallback<GetResponse>() {
          public void run(final GetResponse parameter) {
            System.out.println("Non-Blocking Callback");
            System.out.println(parameter.getDataList());

            stopwatch.stop();
            System.out.printf("Request took %s milliseconds\n", stopwatch.elapsedMillis());
            synchronized (lock) {
              lock.notify();
            }
          }
        });
    synchronized (lock) {
      lock.wait();
    }
  }