public void trackUUID(final UUID uuid, final String name, boolean replace) {
if (uuid != null) {
keys.add(uuid);
if (name != null && name.length() > 0) {
final String keyName = StringUtil.safeString(name);
if (!names.containsKey(keyName)) {
names.put(keyName, uuid);
uuidMap.writeUUIDMap();
} else if (!names.get(keyName).equals(uuid)) {
if (replace) {
ess.getLogger()
.info(
"Found new UUID for "
+ name
+ ". Replacing "
+ names.get(keyName).toString()
+ " with "
+ uuid.toString());
names.put(keyName, uuid);
uuidMap.writeUUIDMap();
} else {
if (ess.getSettings().isDebug()) {
ess.getLogger()
.info(
"Found old UUID for "
+ name
+ " ("
+ uuid.toString()
+ "). Not adding to usermap.");
}
}
}
}
}
}
protected SymbolEvents constructEvent(
String eventListName,
ScreeningSupplementedStock screenedStock,
EventDefinition eventDefinition,
Integer rank,
Integer previousRank,
EventType eventType,
String message,
Date eventDate) {
ConcurrentSkipListMap<EventKey, EventValue> eventMap =
new ConcurrentSkipListMap<EventKey, EventValue>();
EventKey key = new StandardEventKey(eventDate, eventDefinition, eventType);
message = message + ". Rank is : --" + rank + "--";
if (previousRank != null) message = message + " and was " + previousRank;
EventValue value =
new AlertEventValue(eventDate, eventType, eventDefinition, message, eventListName);
eventMap.put(key, value);
SymbolEvents symbolEvents =
new SymbolEvents(
screenedStock.getStock(),
eventMap,
EventDefinition.loadMaxPassPrefsEventInfo(),
EventState.STATE_TERMINATED);
return symbolEvents;
}
/** Test of paintComponent method, of class GraphPanelChart. */
@Test
public void testPaintComponent() {
System.out.println("paintComponent");
Graphics g = new TestGraphics();
GraphPanelChart instance = new GraphPanelChart();
instance.setSize(500, 500);
instance.getChartSettings().setDrawFinalZeroingLines(true);
instance.getChartSettings().setDrawCurrentX(true);
instance.getChartSettings().setExpendRows(true);
final ConcurrentSkipListMap<String, AbstractGraphRow> rows =
new ConcurrentSkipListMap<String, AbstractGraphRow>();
instance.setRows(rows);
final GraphRowAverages row1 = new GraphRowAverages();
row1.setDrawThickLines(true);
row1.setDrawLine(true);
row1.setDrawBar(true);
row1.setDrawValueLabel(true);
row1.setMarkerSize(AbstractGraphRow.MARKER_SIZE_BIG);
rows.put("test 1", row1);
row1.add(System.currentTimeMillis(), 20);
instance.paintComponent(g);
row1.add(System.currentTimeMillis(), 540);
instance.setxAxisLabelRenderer(new DateTimeRenderer("HH:mm:ss"));
instance.paintComponent(g);
row1.add(System.currentTimeMillis(), 8530);
instance.paintComponent(g);
}
synchronized void save() {
Long lastKey = this.lastSyncKey;
Set<Entry<Long, RedoLogValue>> entrySet =
lastKey == null ? skipListMap.entrySet() : skipListMap.tailMap(lastKey, false).entrySet();
if (!entrySet.isEmpty()) {
WriteBuffer buff = WriteBufferPool.poll();
try {
for (Entry<Long, RedoLogValue> e : entrySet) {
lastKey = e.getKey();
keyType.write(buff, lastKey);
valueType.write(buff, e.getValue());
}
int chunkLength = buff.position();
if (chunkLength > 0) {
buff.limit(chunkLength);
buff.position(0);
fileStorage.writeFully(pos, buff.getBuffer());
pos += chunkLength;
fileStorage.sync();
}
this.lastSyncKey = lastKey;
} finally {
WriteBufferPool.offer(buff);
}
}
}
/**
* Adds an old value with a fixed timestamp to the sample.
*
* @param value the value to be added
* @param timestamp the epoch timestamp of {@code value} in seconds
*/
public void update(long value, long timestamp) {
lockForRegularUsage();
try {
final double priority = weight(timestamp - startTime) / random();
final long newCount = count.incrementAndGet();
if (newCount <= reservoirSize) {
values.put(priority, value);
} else {
Double first = values.firstKey();
if (first < priority) {
if (values.putIfAbsent(priority, value) == null) {
// ensure we always remove an item
while (values.remove(first) == null) {
first = values.firstKey();
}
}
}
}
} finally {
unlockForRegularUsage();
}
final long now = System.nanoTime();
final long next = nextScaleTime.get();
if (now >= next) {
rescale(now, next);
}
}
public void fetch() throws IOException {
long start = System.currentTimeMillis();
LOG.info("Processing " + file);
int cnt = 0;
try {
BufferedReader data = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
String line;
while ((line = data.readLine()) != null) {
String[] arr = line.split("\t");
long id = Long.parseLong(arr[0]);
String username = (arr.length > 1) ? arr[1] : "a";
String url = getUrl(id, username);
connections.incrementAndGet();
crawlURL(url, new TweetFetcherHandler(id, username, url, 0, !this.noFollow));
cnt++;
if (cnt % TWEET_BLOCK_SIZE == 0) {
LOG.info(cnt + " requests submitted");
}
}
} catch (IOException e) {
e.printStackTrace();
}
// Wait for the last requests to complete.
LOG.info("Waiting for remaining requests (" + connections.get() + ") to finish!");
while (connections.get() > 0) {
try {
Thread.sleep(1000);
} catch (Exception e) {
e.printStackTrace();
}
}
asyncHttpClient.close();
long end = System.currentTimeMillis();
long duration = end - start;
LOG.info("Total request submitted: " + cnt);
LOG.info(crawl.size() + " tweets fetched in " + duration + "ms");
LOG.info("Writing tweets...");
int written = 0;
Configuration conf = new Configuration();
OutputStreamWriter out =
new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(output)));
for (Map.Entry<Long, String> entry : crawl.entrySet()) {
written++;
out.write(entry.getValue() + "\n");
}
out.close();
LOG.info(written + " statuses written.");
LOG.info("Done!");
}
protected void _run() {
ConcurrentNavigableMap<Long, Entry>
head_map; // head_map = entries which are <= curr time (ready to be executed)
if (!(head_map = tasks.headMap(System.currentTimeMillis(), true)).isEmpty()) {
final List<Long> keys = new LinkedList<Long>();
for (Map.Entry<Long, Entry> entry : head_map.entrySet()) {
final Long key = entry.getKey();
final Entry val = entry.getValue();
pool.execute(
new Runnable() {
public void run() {
val.execute();
}
});
keys.add(key);
}
tasks.keySet().removeAll(keys);
}
if (tasks.isEmpty()) {
no_tasks.compareAndSet(false, true);
waitFor(); // sleeps until time elapses, or a task with a lower execution time is added
} else
waitUntilNextExecution(); // waits until next execution, or a task with a lower execution time
// is added
}
/* Receive a response.
* Because only one receiver, so no synchronization on in.
*/
protected void receiveResponse() {
if (shouldCloseConnection.get()) {
return;
}
touch();
try {
// See HBaseServer.Call.setResponse for where we write out the response.
// It writes the call.id (int), a flag byte, then optionally the length
// of the response (int) followed by data.
// Read the call id.
int id = in.readInt();
if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + id);
Call call = calls.get(id);
// Read the flag byte
byte flag = in.readByte();
boolean isError = ResponseFlag.isError(flag);
if (ResponseFlag.isLength(flag)) {
// Currently length if present is unused.
in.readInt();
}
int state = in.readInt(); // Read the state. Currently unused.
if (isError) {
if (call != null) {
//noinspection ThrowableInstanceNeverThrown
call.setException(
new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)));
}
} else {
Writable value = ReflectionUtils.newInstance(valueClass, conf);
value.readFields(in); // read value
// it's possible that this call may have been cleaned up due to a RPC
// timeout, so check if it still exists before setting the value.
if (call != null) {
call.setValue(value);
}
}
calls.remove(id);
} catch (IOException e) {
if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
// Clean up open calls but don't treat this as a fatal condition,
// since we expect certain responses to not make it by the specified
// {@link ConnectionId#rpcTimeout}.
closeException = e;
} else {
// Since the server did not respond within the default ping interval
// time, treat this as a fatal condition and close this connection
markClosed(e);
}
} finally {
if (remoteId.rpcTimeout > 0) {
cleanupCalls(remoteId.rpcTimeout);
}
}
}
public void open(Serializable e) throws Exception {
Properties jobParameters =
BatchRuntime.getJobOperator().getParameters(jobContext.getExecutionId());
ConcurrentSkipListMap<Integer, PayrollInputRecord> records =
dataBean.getPayrollInputRecords((String) jobParameters.get("monthYear"));
Integer fromKey = (Integer) jobParameters.get("startEmpID");
Integer toKey = (Integer) jobParameters.get("endEmpID");
payrollInputRecords = records.subMap(fromKey, true, toKey, false).values().iterator();
}
public void offer(int sid, int trunkId, int segId, Object data) {
TrunkSegmentIdentifier id = new TrunkSegmentIdentifier(msgId, sid, trunkId, segId);
while (!cacheQueue.containsKey(id) && cacheQueue.size() > maxQueueItems) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
cacheQueue.put(id, data);
msgId++;
}
public void removeUser(final String name) {
if (names == null) {
ess.getLogger().warning("Name collection is null, cannot remove user.");
return;
}
UUID uuid = names.get(name);
if (uuid != null) {
keys.remove(uuid);
users.invalidate(uuid);
}
names.remove(name);
names.remove(StringUtil.safeString(name));
}
public OCachePointer load(long fileId, long pageIndex) throws IOException {
synchronized (syncObject) {
final GroupKey groupKey = new GroupKey(fileId, pageIndex >>> 4);
lockManager.acquireLock(Thread.currentThread(), groupKey, OLockManager.LOCK.SHARED);
try {
final WriteGroup writeGroup = writeGroups.get(groupKey);
OCachePointer pagePointer;
if (writeGroup == null) {
pagePointer = cacheFileContent(fileId, pageIndex);
pagePointer.incrementReferrer();
return pagePointer;
}
final int entryIndex = (int) (pageIndex & 15);
pagePointer = writeGroup.pages[entryIndex];
if (pagePointer == null) pagePointer = cacheFileContent(fileId, pageIndex);
pagePointer.incrementReferrer();
return pagePointer;
} finally {
lockManager.releaseLock(Thread.currentThread(), groupKey, OLockManager.LOCK.SHARED);
}
}
}
/* "A common feature of the above techniques—indeed, the key technique that
* allows us to track the decayed weights efficiently—is that they maintain
* counts and other quantities based on g(ti − L), and only scale by g(t − L)
* at query time. But while g(ti −L)/g(t−L) is guaranteed to lie between zero
* and one, the intermediate values of g(ti − L) could become very large. For
* polynomial functions, these values should not grow too large, and should be
* effectively represented in practice by floating point values without loss of
* precision. For exponential functions, these values could grow quite large as
* new values of (ti − L) become large, and potentially exceed the capacity of
* common floating point types. However, since the values stored by the
* algorithms are linear combinations of g values (scaled sums), they can be
* rescaled relative to a new landmark. That is, by the analysis of exponential
* decay in Section III-A, the choice of L does not affect the final result. We
* can therefore multiply each value based on L by a factor of exp(−α(L′ − L)),
* and obtain the correct value as if we had instead computed relative to a new
* landmark L′ (and then use this new L′ at query time). This can be done with
* a linear pass over whatever data structure is being used."
*/
private void rescale(long now, long next) {
if (nextScaleTime.compareAndSet(next, now + RESCALE_THRESHOLD)) {
lockForRescale();
try {
final long oldStartTime = startTime;
this.startTime = tick();
final ArrayList<Double> keys = new ArrayList<Double>(values.keySet());
for (Double key : keys) {
final Long value = values.remove(key);
values.put(key * exp(-alpha * (startTime - oldStartTime)), value);
}
} finally {
unlockForRescale();
}
}
}
public HistoryFileInfo remove(JobId key) {
HistoryFileInfo ret = cache.remove(key);
if (ret != null) {
mapSize.decrementAndGet();
}
return ret;
}
public String dumpTimerTasks() {
StringBuilder sb = new StringBuilder();
for (Entry entry : tasks.values()) {
sb.append(entry.dump()).append("\n");
}
return sb.toString();
}
public HistoryFileInfo putIfAbsent(JobId key, HistoryFileInfo value) {
HistoryFileInfo ret = cache.putIfAbsent(key, value);
if (ret == null) {
mapSize.incrementAndGet();
}
return ret;
}
Long lastKey() {
try {
return skipListMap.lastKey();
} catch (NoSuchElementException e) {
return null;
}
}
public List<Cluster> getAllCluster() {
List<Cluster> cl = new ArrayList<Cluster>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.CLUSTER_PATH_S, true, PathUtil.CLUSTER_PATH_E, true).entrySet()) {
cl.add((Cluster) e.getValue());
}
return cl;
}
public List<Long> values() {
lockForRegularUsage();
try {
return new ArrayList<Long>(values.values());
} finally {
unlockForRegularUsage();
}
}
public Object pop() {
Map.Entry e = cacheQueue.pollFirstEntry();
if (e != null) {
return e.getValue();
} else {
return null;
}
}
public List<PhysicalMachine> getAllPm() {
List<PhysicalMachine> pml = new ArrayList<PhysicalMachine>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.PM_PATH_S, true, PathUtil.PM_PATH_E, true).entrySet()) {
pml.add((PhysicalMachine) e.getValue());
}
return pml;
}
synchronized void write(Source source) {
if (source == null) return;
// StringBuilder builder = new StringBuilder(source.getName()).append('.');
// builder.append(source.getGroup()).append('.').append(source.getCategory());
int hashCode = trackerService.hashCode(source);
// System.out.println(hashCode+ " : " + Calendar.getInstance().getTimeInMillis());
codes.put(hashCode, Calendar.getInstance().getTimeInMillis());
}
public List<PartitionGroupServer> getPgsList(String clusterName) {
List<PartitionGroupServer> pgsl = new ArrayList<PartitionGroupServer>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.PGS_PATH_S(clusterName), true, PathUtil.PGS_PATH_E(clusterName), true)
.entrySet()) {
pgsl.add((PartitionGroupServer) e.getValue());
}
return pgsl;
}
private void read() {
ByteBuffer buffer = fileStorage.readFully(0, (int) pos);
while (buffer.remaining() > 0) {
Long k = (Long) keyType.read(buffer);
RedoLogValue v = (RedoLogValue) valueType.read(buffer);
skipListMap.put(k, v);
lastSyncKey = k;
}
}
@Override
public Snapshot getSnapshot() {
lockForRegularUsage();
try {
return new WeightedSnapshot(values.values());
} finally {
unlockForRegularUsage();
}
}
public List<PhysicalMachineCluster> getPmcList(String pmName) {
List<PhysicalMachineCluster> pmcl = new ArrayList<PhysicalMachineCluster>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.PMC_PATH_S(pmName), true, PathUtil.PMC_PATH_E(pmName), true)
.entrySet()) {
pmcl.add((PhysicalMachineCluster) e.getValue());
}
return pmcl;
}
public List<Gateway> getGwList(String clusterName) {
List<Gateway> gwl = new ArrayList<Gateway>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.GW_PATH_S(clusterName), true, PathUtil.GW_PATH_E(clusterName), true)
.entrySet()) {
gwl.add((Gateway) e.getValue());
}
return gwl;
}
public List<RedisServer> getRsList(String clusterName) {
List<RedisServer> rsl = new ArrayList<RedisServer>();
for (Entry<String, ClusterComponent> e :
map.subMap(PathUtil.RS_PATH_S(clusterName), true, PathUtil.RS_PATH_E(clusterName), true)
.entrySet()) {
rsl.add((RedisServer) e.getValue());
}
return rsl;
}
protected void cleanupCalls(long rpcTimeout) {
Iterator<Entry<Integer, Call>> itor = calls.entrySet().iterator();
while (itor.hasNext()) {
Call c = itor.next().getValue();
long waitTime = System.currentTimeMillis() - c.getStartTime();
if (waitTime >= rpcTimeout) {
if (this.closeException == null) {
// There may be no exception in the case that there are many calls
// being multiplexed over this connection and these are succeeding
// fine while this Call object is taking a long time to finish
// over on the server; e.g. I just asked the regionserver to bulk
// open 3k regions or its a big fat multiput into a heavily-loaded
// server (Perhaps this only happens at the extremes?)
this.closeException =
new CallTimeoutException(
"Call id=" + c.id + ", waitTime=" + waitTime + ", rpcTimetout=" + rpcTimeout);
}
c.setException(this.closeException);
synchronized (c) {
c.notifyAll();
}
itor.remove();
} else {
break;
}
}
try {
if (!calls.isEmpty()) {
Call firstCall = calls.get(calls.firstKey());
long maxWaitTime = System.currentTimeMillis() - firstCall.getStartTime();
if (maxWaitTime < rpcTimeout) {
rpcTimeout -= maxWaitTime;
}
}
if (!shouldCloseConnection.get()) {
closeException = null;
if (socket != null) {
socket.setSoTimeout((int) rpcTimeout);
}
}
} catch (SocketException e) {
LOG.debug("Couldn't lower timeout, which may result in longer than expected calls");
}
}
/** Create a map from Integers -5 to -1 to Strings "A"-"E". */
private static ConcurrentNavigableMap dmap5() {
ConcurrentSkipListMap map = new ConcurrentSkipListMap();
assertTrue(map.isEmpty());
map.put(m1, "A");
map.put(m5, "E");
map.put(m3, "C");
map.put(m2, "B");
map.put(m4, "D");
assertFalse(map.isEmpty());
assertEquals(5, map.size());
return map.descendingMap();
}