Exemple #1
0
 public void trackUUID(final UUID uuid, final String name, boolean replace) {
   if (uuid != null) {
     keys.add(uuid);
     if (name != null && name.length() > 0) {
       final String keyName = StringUtil.safeString(name);
       if (!names.containsKey(keyName)) {
         names.put(keyName, uuid);
         uuidMap.writeUUIDMap();
       } else if (!names.get(keyName).equals(uuid)) {
         if (replace) {
           ess.getLogger()
               .info(
                   "Found new UUID for "
                       + name
                       + ". Replacing "
                       + names.get(keyName).toString()
                       + " with "
                       + uuid.toString());
           names.put(keyName, uuid);
           uuidMap.writeUUIDMap();
         } else {
           if (ess.getSettings().isDebug()) {
             ess.getLogger()
                 .info(
                     "Found old UUID for "
                         + name
                         + " ("
                         + uuid.toString()
                         + "). Not adding to usermap.");
           }
         }
       }
     }
   }
 }
  protected SymbolEvents constructEvent(
      String eventListName,
      ScreeningSupplementedStock screenedStock,
      EventDefinition eventDefinition,
      Integer rank,
      Integer previousRank,
      EventType eventType,
      String message,
      Date eventDate) {

    ConcurrentSkipListMap<EventKey, EventValue> eventMap =
        new ConcurrentSkipListMap<EventKey, EventValue>();

    EventKey key = new StandardEventKey(eventDate, eventDefinition, eventType);
    message = message + ". Rank is : --" + rank + "--";
    if (previousRank != null) message = message + " and was " + previousRank;
    EventValue value =
        new AlertEventValue(eventDate, eventType, eventDefinition, message, eventListName);
    eventMap.put(key, value);
    SymbolEvents symbolEvents =
        new SymbolEvents(
            screenedStock.getStock(),
            eventMap,
            EventDefinition.loadMaxPassPrefsEventInfo(),
            EventState.STATE_TERMINATED);
    return symbolEvents;
  }
  /** Test of paintComponent method, of class GraphPanelChart. */
  @Test
  public void testPaintComponent() {
    System.out.println("paintComponent");
    Graphics g = new TestGraphics();
    GraphPanelChart instance = new GraphPanelChart();
    instance.setSize(500, 500);
    instance.getChartSettings().setDrawFinalZeroingLines(true);
    instance.getChartSettings().setDrawCurrentX(true);
    instance.getChartSettings().setExpendRows(true);

    final ConcurrentSkipListMap<String, AbstractGraphRow> rows =
        new ConcurrentSkipListMap<String, AbstractGraphRow>();
    instance.setRows(rows);
    final GraphRowAverages row1 = new GraphRowAverages();
    row1.setDrawThickLines(true);
    row1.setDrawLine(true);
    row1.setDrawBar(true);
    row1.setDrawValueLabel(true);
    row1.setMarkerSize(AbstractGraphRow.MARKER_SIZE_BIG);
    rows.put("test 1", row1);
    row1.add(System.currentTimeMillis(), 20);

    instance.paintComponent(g);

    row1.add(System.currentTimeMillis(), 540);
    instance.setxAxisLabelRenderer(new DateTimeRenderer("HH:mm:ss"));
    instance.paintComponent(g);

    row1.add(System.currentTimeMillis(), 8530);
    instance.paintComponent(g);
  }
 synchronized void save() {
   Long lastKey = this.lastSyncKey;
   Set<Entry<Long, RedoLogValue>> entrySet =
       lastKey == null ? skipListMap.entrySet() : skipListMap.tailMap(lastKey, false).entrySet();
   if (!entrySet.isEmpty()) {
     WriteBuffer buff = WriteBufferPool.poll();
     try {
       for (Entry<Long, RedoLogValue> e : entrySet) {
         lastKey = e.getKey();
         keyType.write(buff, lastKey);
         valueType.write(buff, e.getValue());
       }
       int chunkLength = buff.position();
       if (chunkLength > 0) {
         buff.limit(chunkLength);
         buff.position(0);
         fileStorage.writeFully(pos, buff.getBuffer());
         pos += chunkLength;
         fileStorage.sync();
       }
       this.lastSyncKey = lastKey;
     } finally {
       WriteBufferPool.offer(buff);
     }
   }
 }
  /**
   * Adds an old value with a fixed timestamp to the sample.
   *
   * @param value the value to be added
   * @param timestamp the epoch timestamp of {@code value} in seconds
   */
  public void update(long value, long timestamp) {
    lockForRegularUsage();
    try {
      final double priority = weight(timestamp - startTime) / random();
      final long newCount = count.incrementAndGet();
      if (newCount <= reservoirSize) {
        values.put(priority, value);
      } else {
        Double first = values.firstKey();
        if (first < priority) {
          if (values.putIfAbsent(priority, value) == null) {
            // ensure we always remove an item
            while (values.remove(first) == null) {
              first = values.firstKey();
            }
          }
        }
      }
    } finally {
      unlockForRegularUsage();
    }

    final long now = System.nanoTime();
    final long next = nextScaleTime.get();
    if (now >= next) {
      rescale(now, next);
    }
  }
  public void fetch() throws IOException {
    long start = System.currentTimeMillis();
    LOG.info("Processing " + file);

    int cnt = 0;
    try {
      BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
      String line;
      while ((line = data.readLine()) != null) {
        String[] arr = line.split("\t");
        long id = Long.parseLong(arr[0]);
        String username = (arr.length > 1) ? arr[1] : "a";
        String url = getUrl(id, username);

        connections.incrementAndGet();
        crawlURL(url, new TweetFetcherHandler(id, username, url, 0, !this.noFollow));

        cnt++;

        if (cnt % TWEET_BLOCK_SIZE == 0) {
          LOG.info(cnt + " requests submitted");
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    // Wait for the last requests to complete.
    LOG.info("Waiting for remaining requests (" + connections.get() + ") to finish!");
    while (connections.get() > 0) {
      try {
        Thread.sleep(1000);
      } catch (Exception e) {
        e.printStackTrace();
      }
    }

    asyncHttpClient.close();

    long end = System.currentTimeMillis();
    long duration = end - start;
    LOG.info("Total request submitted: " + cnt);
    LOG.info(crawl.size() + " tweets fetched in " + duration + "ms");

    LOG.info("Writing tweets...");
    int written = 0;
    Configuration conf = new Configuration();

    OutputStreamWriter out =
        new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(output)));
    for (Map.Entry<Long, String> entry : crawl.entrySet()) {
      written++;
      out.write(entry.getValue() + "\n");
    }

    out.close();

    LOG.info(written + " statuses written.");
    LOG.info("Done!");
  }
Exemple #7
0
  protected void _run() {
    ConcurrentNavigableMap<Long, Entry>
        head_map; // head_map = entries which are <= curr time (ready to be executed)
    if (!(head_map = tasks.headMap(System.currentTimeMillis(), true)).isEmpty()) {
      final List<Long> keys = new LinkedList<Long>();
      for (Map.Entry<Long, Entry> entry : head_map.entrySet()) {
        final Long key = entry.getKey();
        final Entry val = entry.getValue();
        pool.execute(
            new Runnable() {
              public void run() {
                val.execute();
              }
            });
        keys.add(key);
      }
      tasks.keySet().removeAll(keys);
    }

    if (tasks.isEmpty()) {
      no_tasks.compareAndSet(false, true);
      waitFor(); // sleeps until time elapses, or a task with a lower execution time is added
    } else
      waitUntilNextExecution(); // waits until next execution, or a task with a lower execution time
    // is added
  }
Exemple #8
0
    /* Receive a response.
     * Because only one receiver, so no synchronization on in.
     */
    protected void receiveResponse() {
      if (shouldCloseConnection.get()) {
        return;
      }
      touch();

      try {
        // See HBaseServer.Call.setResponse for where we write out the response.
        // It writes the call.id (int), a flag byte, then optionally the length
        // of the response (int) followed by data.

        // Read the call id.
        int id = in.readInt();

        if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + id);
        Call call = calls.get(id);

        // Read the flag byte
        byte flag = in.readByte();
        boolean isError = ResponseFlag.isError(flag);
        if (ResponseFlag.isLength(flag)) {
          // Currently length if present is unused.
          in.readInt();
        }
        int state = in.readInt(); // Read the state.  Currently unused.
        if (isError) {
          if (call != null) {
            //noinspection ThrowableInstanceNeverThrown
            call.setException(
                new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)));
          }
        } else {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in); // read value
          // it's possible that this call may have been cleaned up due to a RPC
          // timeout, so check if it still exists before setting the value.
          if (call != null) {
            call.setValue(value);
          }
        }
        calls.remove(id);
      } catch (IOException e) {
        if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
          // Clean up open calls but don't treat this as a fatal condition,
          // since we expect certain responses to not make it by the specified
          // {@link ConnectionId#rpcTimeout}.
          closeException = e;
        } else {
          // Since the server did not respond within the default ping interval
          // time, treat this as a fatal condition and close this connection
          markClosed(e);
        }
      } finally {
        if (remoteId.rpcTimeout > 0) {
          cleanupCalls(remoteId.rpcTimeout);
        }
      }
    }
 public void open(Serializable e) throws Exception {
   Properties jobParameters =
       BatchRuntime.getJobOperator().getParameters(jobContext.getExecutionId());
   ConcurrentSkipListMap<Integer, PayrollInputRecord> records =
       dataBean.getPayrollInputRecords((String) jobParameters.get("monthYear"));
   Integer fromKey = (Integer) jobParameters.get("startEmpID");
   Integer toKey = (Integer) jobParameters.get("endEmpID");
   payrollInputRecords = records.subMap(fromKey, true, toKey, false).values().iterator();
 }
 public void offer(int sid, int trunkId, int segId, Object data) {
   TrunkSegmentIdentifier id = new TrunkSegmentIdentifier(msgId, sid, trunkId, segId);
   while (!cacheQueue.containsKey(id) && cacheQueue.size() > maxQueueItems) {
     try {
       Thread.sleep(10);
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
   }
   cacheQueue.put(id, data);
   msgId++;
 }
Exemple #11
0
 public void removeUser(final String name) {
   if (names == null) {
     ess.getLogger().warning("Name collection is null, cannot remove user.");
     return;
   }
   UUID uuid = names.get(name);
   if (uuid != null) {
     keys.remove(uuid);
     users.invalidate(uuid);
   }
   names.remove(name);
   names.remove(StringUtil.safeString(name));
 }
Exemple #12
0
  public OCachePointer load(long fileId, long pageIndex) throws IOException {
    synchronized (syncObject) {
      final GroupKey groupKey = new GroupKey(fileId, pageIndex >>> 4);
      lockManager.acquireLock(Thread.currentThread(), groupKey, OLockManager.LOCK.SHARED);
      try {
        final WriteGroup writeGroup = writeGroups.get(groupKey);

        OCachePointer pagePointer;
        if (writeGroup == null) {
          pagePointer = cacheFileContent(fileId, pageIndex);
          pagePointer.incrementReferrer();

          return pagePointer;
        }

        final int entryIndex = (int) (pageIndex & 15);
        pagePointer = writeGroup.pages[entryIndex];

        if (pagePointer == null) pagePointer = cacheFileContent(fileId, pageIndex);

        pagePointer.incrementReferrer();
        return pagePointer;
      } finally {
        lockManager.releaseLock(Thread.currentThread(), groupKey, OLockManager.LOCK.SHARED);
      }
    }
  }
 /* "A common feature of the above techniques—indeed, the key technique that
  * allows us to track the decayed weights efficiently—is that they maintain
  * counts and other quantities based on g(ti − L), and only scale by g(t − L)
  * at query time. But while g(ti −L)/g(t−L) is guaranteed to lie between zero
  * and one, the intermediate values of g(ti − L) could become very large. For
  * polynomial functions, these values should not grow too large, and should be
  * effectively represented in practice by floating point values without loss of
  * precision. For exponential functions, these values could grow quite large as
  * new values of (ti − L) become large, and potentially exceed the capacity of
  * common floating point types. However, since the values stored by the
  * algorithms are linear combinations of g values (scaled sums), they can be
  * rescaled relative to a new landmark. That is, by the analysis of exponential
  * decay in Section III-A, the choice of L does not affect the final result. We
  * can therefore multiply each value based on L by a factor of exp(−α(L′ − L)),
  * and obtain the correct value as if we had instead computed relative to a new
  * landmark L′ (and then use this new L′ at query time). This can be done with
  * a linear pass over whatever data structure is being used."
  */
 private void rescale(long now, long next) {
   if (nextScaleTime.compareAndSet(next, now + RESCALE_THRESHOLD)) {
     lockForRescale();
     try {
       final long oldStartTime = startTime;
       this.startTime = tick();
       final ArrayList<Double> keys = new ArrayList<Double>(values.keySet());
       for (Double key : keys) {
         final Long value = values.remove(key);
         values.put(key * exp(-alpha * (startTime - oldStartTime)), value);
       }
     } finally {
       unlockForRescale();
     }
   }
 }
 public HistoryFileInfo remove(JobId key) {
   HistoryFileInfo ret = cache.remove(key);
   if (ret != null) {
     mapSize.decrementAndGet();
   }
   return ret;
 }
Exemple #15
0
 public String dumpTimerTasks() {
   StringBuilder sb = new StringBuilder();
   for (Entry entry : tasks.values()) {
     sb.append(entry.dump()).append("\n");
   }
   return sb.toString();
 }
 public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
   HistoryFileInfo ret = cache.putIfAbsent(key, value);
   if (ret == null) {
     mapSize.incrementAndGet();
   }
   return ret;
 }
 Long lastKey() {
   try {
     return skipListMap.lastKey();
   } catch (NoSuchElementException e) {
     return null;
   }
 }
 public List<Cluster> getAllCluster() {
   List<Cluster> cl = new ArrayList<Cluster>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.CLUSTER_PATH_S, true, PathUtil.CLUSTER_PATH_E, true).entrySet()) {
     cl.add((Cluster) e.getValue());
   }
   return cl;
 }
 public List<Long> values() {
   lockForRegularUsage();
   try {
     return new ArrayList<Long>(values.values());
   } finally {
     unlockForRegularUsage();
   }
 }
 public Object pop() {
   Map.Entry e = cacheQueue.pollFirstEntry();
   if (e != null) {
     return e.getValue();
   } else {
     return null;
   }
 }
 public List<PhysicalMachine> getAllPm() {
   List<PhysicalMachine> pml = new ArrayList<PhysicalMachine>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.PM_PATH_S, true, PathUtil.PM_PATH_E, true).entrySet()) {
     pml.add((PhysicalMachine) e.getValue());
   }
   return pml;
 }
Exemple #22
0
 synchronized void write(Source source) {
   if (source == null) return;
   //    StringBuilder builder = new StringBuilder(source.getName()).append('.');
   //    builder.append(source.getGroup()).append('.').append(source.getCategory());
   int hashCode = trackerService.hashCode(source);
   //    System.out.println(hashCode+ "  : " + Calendar.getInstance().getTimeInMillis());
   codes.put(hashCode, Calendar.getInstance().getTimeInMillis());
 }
 public List<PartitionGroupServer> getPgsList(String clusterName) {
   List<PartitionGroupServer> pgsl = new ArrayList<PartitionGroupServer>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.PGS_PATH_S(clusterName), true, PathUtil.PGS_PATH_E(clusterName), true)
           .entrySet()) {
     pgsl.add((PartitionGroupServer) e.getValue());
   }
   return pgsl;
 }
 private void read() {
   ByteBuffer buffer = fileStorage.readFully(0, (int) pos);
   while (buffer.remaining() > 0) {
     Long k = (Long) keyType.read(buffer);
     RedoLogValue v = (RedoLogValue) valueType.read(buffer);
     skipListMap.put(k, v);
     lastSyncKey = k;
   }
 }
 @Override
 public Snapshot getSnapshot() {
   lockForRegularUsage();
   try {
     return new WeightedSnapshot(values.values());
   } finally {
     unlockForRegularUsage();
   }
 }
 public List<PhysicalMachineCluster> getPmcList(String pmName) {
   List<PhysicalMachineCluster> pmcl = new ArrayList<PhysicalMachineCluster>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.PMC_PATH_S(pmName), true, PathUtil.PMC_PATH_E(pmName), true)
           .entrySet()) {
     pmcl.add((PhysicalMachineCluster) e.getValue());
   }
   return pmcl;
 }
 public List<Gateway> getGwList(String clusterName) {
   List<Gateway> gwl = new ArrayList<Gateway>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.GW_PATH_S(clusterName), true, PathUtil.GW_PATH_E(clusterName), true)
           .entrySet()) {
     gwl.add((Gateway) e.getValue());
   }
   return gwl;
 }
 public List<RedisServer> getRsList(String clusterName) {
   List<RedisServer> rsl = new ArrayList<RedisServer>();
   for (Entry<String, ClusterComponent> e :
       map.subMap(PathUtil.RS_PATH_S(clusterName), true, PathUtil.RS_PATH_E(clusterName), true)
           .entrySet()) {
     rsl.add((RedisServer) e.getValue());
   }
   return rsl;
 }
Exemple #29
0
 protected void cleanupCalls(long rpcTimeout) {
   Iterator<Entry<Integer, Call>> itor = calls.entrySet().iterator();
   while (itor.hasNext()) {
     Call c = itor.next().getValue();
     long waitTime = System.currentTimeMillis() - c.getStartTime();
     if (waitTime >= rpcTimeout) {
       if (this.closeException == null) {
         // There may be no exception in the case that there are many calls
         // being multiplexed over this connection and these are succeeding
         // fine while this Call object is taking a long time to finish
         // over on the server; e.g. I just asked the regionserver to bulk
         // open 3k regions or its a big fat multiput into a heavily-loaded
         // server (Perhaps this only happens at the extremes?)
         this.closeException =
             new CallTimeoutException(
                 "Call id=" + c.id + ", waitTime=" + waitTime + ", rpcTimetout=" + rpcTimeout);
       }
       c.setException(this.closeException);
       synchronized (c) {
         c.notifyAll();
       }
       itor.remove();
     } else {
       break;
     }
   }
   try {
     if (!calls.isEmpty()) {
       Call firstCall = calls.get(calls.firstKey());
       long maxWaitTime = System.currentTimeMillis() - firstCall.getStartTime();
       if (maxWaitTime < rpcTimeout) {
         rpcTimeout -= maxWaitTime;
       }
     }
     if (!shouldCloseConnection.get()) {
       closeException = null;
       if (socket != null) {
         socket.setSoTimeout((int) rpcTimeout);
       }
     }
   } catch (SocketException e) {
     LOG.debug("Couldn't lower timeout, which may result in longer than expected calls");
   }
 }
 /** Create a map from Integers -5 to -1 to Strings "A"-"E". */
 private static ConcurrentNavigableMap dmap5() {
   ConcurrentSkipListMap map = new ConcurrentSkipListMap();
   assertTrue(map.isEmpty());
   map.put(m1, "A");
   map.put(m5, "E");
   map.put(m3, "C");
   map.put(m2, "B");
   map.put(m4, "D");
   assertFalse(map.isEmpty());
   assertEquals(5, map.size());
   return map.descendingMap();
 }