private String getAvgTimePerTick(SortedMap<Long, Record> records) {
long sum = 0;
for (Map.Entry<Long, Record> en : records.entrySet()) {
sum += en.getValue().getLastCallDiff();
}
long globalavg = sum / records.size();
sum = 0;
Set<Map.Entry<Long, Record>> subset =
records.tailMap(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(15)).entrySet();
for (Map.Entry<Long, Record> en : subset) {
sum += en.getValue().getLastCallDiff();
}
long min15avg = sum / subset.size();
sum = 0;
subset = records.tailMap(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(10)).entrySet();
for (Map.Entry<Long, Record> en : subset) {
sum += en.getValue().getLastCallDiff();
}
long min10avg = sum / subset.size();
sum = 0;
subset = records.tailMap(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5)).entrySet();
for (Map.Entry<Long, Record> en : subset) {
sum += en.getValue().getLastCallDiff();
}
long min5avg = sum / subset.size();
sum = 0;
subset = records.tailMap(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1)).entrySet();
for (Map.Entry<Long, Record> en : subset) {
sum += en.getValue().getLastCallDiff();
}
long min1avg = sum / subset.size();
return globalavg
+ " "
+ ChatColor.GRAY
+ min15avg
+ " "
+ ChatColor.YELLOW
+ min10avg
+ " "
+ ChatColor.GOLD
+ min5avg
+ " "
+ ChatColor.GREEN
+ min1avg;
}
/**
* rename named record into newName
*
* @param oldName current name of record/collection
* @param newName new name of record/collection
* @throws NoSuchElementException if oldName does not exist
*/
public synchronized void rename(String oldName, String newName) {
if (oldName.equals(newName)) return;
Map<String, Object> sub = catalog.tailMap(oldName);
List<String> toRemove = new ArrayList<String>();
for (String param : sub.keySet()) {
if (!param.startsWith(oldName)) break;
String suffix = param.substring(oldName.length());
catalog.put(newName + suffix, catalog.get(param));
toRemove.add(param);
}
if (toRemove.isEmpty())
throw new NoSuchElementException("Could not rename, name does not exist: " + oldName);
WeakReference old = namesInstanciated.remove(oldName);
if (old != null) {
Object old2 = old.get();
if (old2 != null) {
namesLookup.remove(old2);
namedPut(newName, old2);
}
}
for (String param : toRemove) catalog.remove(param);
}
public boolean next(SubRequest request) {
MOScope scope = request.getQuery().getScope();
SortedMap<OID, Variable> tail = vbs.tailMap(scope.getLowerBound());
OID first = tail.firstKey();
if (scope.getLowerBound().equals(first) && (!scope.isLowerIncluded())) {
if (tail.size() > 1) {
Iterator<OID> it = tail.keySet().iterator();
it.next();
first = it.next();
} else {
return false;
}
}
if (first != null) {
Variable vb = vbs.get(first);
if (vb == null) {
request.getVariableBinding().setVariable(Null.noSuchInstance);
}
request.getVariableBinding().setOid(first);
request.getVariableBinding().setVariable(vb);
request.completed();
return true;
}
return false;
}
/** Creates a sub map by using the given range (both ends inclusive). */
public static <V> SortedMap<Integer, V> filter(
SortedMap<Integer, V> map, String from, String to) {
if (from == null && to == null) return map;
if (to == null) return map.headMap(Integer.parseInt(from) - 1);
if (from == null) return map.tailMap(Integer.parseInt(to));
return map.subMap(Integer.parseInt(to), Integer.parseInt(from) - 1);
}
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
iterator = terms.tailMap(text).entrySet().iterator();
if (!iterator.hasNext()) {
return SeekStatus.END;
} else {
return next().equals(text) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
}
}
public List<ChatMessage> getAllRoomMessagesAfter(final ChatMessage aLastMsg) {
final SortedMap<Date, ChatMessage> map = getMapForRoom(aLastMsg.getRoom());
synchronized (map) {
final List<ChatMessage> ret = sanitizeMap(map.tailMap(aLastMsg.getDate()));
if (ret.get(0).getDate().equals(aLastMsg.getDate())) {
ret.remove(0);
}
return ret;
}
}
public T get(Object key) {
if (circle.isEmpty()) {
return null;
}
int hash = hashFunction.hash(key);
if (!circle.containsKey(hash)) {
SortedMap<Integer, T> tailMap = circle.tailMap(hash);
hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey();
}
return circle.get(hash);
}
private OperationsNodeInfo getNearest(byte[] hash) {
if (circle.isEmpty()) {
return null;
}
if (circle.size() == 1) {
return circle.get(circle.firstKey());
}
SortedMap<byte[], OperationsNodeInfo> tailMap = circle.tailMap(hash);
hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey();
return circle.get(hash);
}
public OID find(MOScope range) {
SortedMap<OID, Variable> tail = vbs.tailMap(range.getLowerBound());
OID first = tail.firstKey();
if (range.getLowerBound().equals(first) && (!range.isLowerIncluded())) {
if (tail.size() > 1) {
Iterator<OID> it = tail.keySet().iterator();
it.next();
return it.next();
}
} else {
return first;
}
return null;
}
public IContainer next() {
String key = containers.firstKey();
IContainer result = containers.get(key);
containers.remove(key);
key += "/";
// Remove child containers
SortedMap<String, IContainer> childs = containers.tailMap(key);
for (Iterator<String> it = childs.keySet().iterator(); it.hasNext(); ) {
if (it.next().startsWith(key)) it.remove();
else break;
}
return result;
}
@Override
public SocketAddress getServer(Object key) {
int keyHashCode = getNormalizedHash(key);
if (keyHashCode == Integer.MIN_VALUE) keyHashCode += 1;
int hash = Math.abs(keyHashCode);
SortedMap<Integer, SocketAddress> candidates = positions.tailMap(hash % hashSpace);
if (log.isTraceEnabled()) {
log.tracef("Found possible candidates: %s", candidates);
}
return (candidates.size() > 0 ? candidates : positions)
.entrySet()
.iterator()
.next()
.getValue();
}
protected Token getOrAddTerminalNode(int index) throws MaltChainedException {
Token node = null;
if (!terminalNodes.containsKey(index)) {
if (index > 0) {
node = terminalPool.checkOut();
node.setIndex(index);
node.setBelongsToGraph(this);
if (index > 1) {
Token prev = terminalNodes.get(index - 1);
if (prev == null) {
try {
prev = terminalNodes.get(terminalNodes.headMap(index).lastKey());
} catch (NoSuchElementException e) {
}
}
if (prev != null) {
prev.setSuccessor(node);
node.setPredecessor(prev);
}
if (terminalNodes.lastKey() > index) {
Token succ = terminalNodes.get(index + 1);
if (succ == null) {
try {
succ = terminalNodes.get(terminalNodes.tailMap(index).firstKey());
} catch (NoSuchElementException e) {
}
}
if (succ != null) {
succ.setPredecessor(node);
node.setSuccessor(succ);
}
}
}
}
terminalNodes.put(index, node);
numberOfComponents++;
} else {
node = terminalNodes.get(index);
}
return node;
}
/** headMap returns map with keys in requested range */
public void testDescendingTailMapContents() {
ConcurrentNavigableMap map = dmap5();
SortedMap sm = map.tailMap(m2);
assertFalse(sm.containsKey(m1));
assertTrue(sm.containsKey(m2));
assertTrue(sm.containsKey(m3));
assertTrue(sm.containsKey(m4));
assertTrue(sm.containsKey(m5));
Iterator i = sm.keySet().iterator();
Object k;
k = (Integer) (i.next());
assertEquals(m2, k);
k = (Integer) (i.next());
assertEquals(m3, k);
k = (Integer) (i.next());
assertEquals(m4, k);
k = (Integer) (i.next());
assertEquals(m5, k);
assertFalse(i.hasNext());
Iterator ei = sm.entrySet().iterator();
Map.Entry e;
e = (Map.Entry) (ei.next());
assertEquals(m2, e.getKey());
assertEquals("B", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(m3, e.getKey());
assertEquals("C", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(m4, e.getKey());
assertEquals("D", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(m5, e.getKey());
assertEquals("E", e.getValue());
assertFalse(i.hasNext());
SortedMap ssm = sm.tailMap(m4);
assertEquals(m4, ssm.firstKey());
assertEquals(m5, ssm.lastKey());
assertEquals("D", ssm.remove(m4));
assertEquals(1, ssm.size());
assertEquals(3, sm.size());
assertEquals(4, map.size());
}
/** headMap returns map with keys in requested range */
public void testTailMapContents() {
ConcurrentNavigableMap map = map5();
SortedMap sm = map.tailMap(two);
assertFalse(sm.containsKey(one));
assertTrue(sm.containsKey(two));
assertTrue(sm.containsKey(three));
assertTrue(sm.containsKey(four));
assertTrue(sm.containsKey(five));
Iterator i = sm.keySet().iterator();
Object k;
k = (Integer) (i.next());
assertEquals(two, k);
k = (Integer) (i.next());
assertEquals(three, k);
k = (Integer) (i.next());
assertEquals(four, k);
k = (Integer) (i.next());
assertEquals(five, k);
assertFalse(i.hasNext());
Iterator ei = sm.entrySet().iterator();
Map.Entry e;
e = (Map.Entry) (ei.next());
assertEquals(two, e.getKey());
assertEquals("B", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(three, e.getKey());
assertEquals("C", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(four, e.getKey());
assertEquals("D", e.getValue());
e = (Map.Entry) (ei.next());
assertEquals(five, e.getKey());
assertEquals("E", e.getValue());
assertFalse(i.hasNext());
SortedMap ssm = sm.tailMap(four);
assertEquals(four, ssm.firstKey());
assertEquals(five, ssm.lastKey());
assertEquals("D", ssm.remove(four));
assertEquals(1, ssm.size());
assertEquals(3, sm.size());
assertEquals(4, map.size());
}
/**
* Caches. Fails silently.
*
* @param event matching this event only, case sensitive
* @param since since this time, 0 for all
* @return non-null, Map of times to (possibly empty) info strings, sorted, earliest first,
* unmodifiable
*/
public synchronized SortedMap<Long, String> getEvents(String event, long since) {
SortedMap<Long, String> rv = _cache.get(event);
if (rv != null) {
Long cacheTime = _cacheTime.get(event);
if (cacheTime != null) {
if (since >= cacheTime.longValue()) return rv.tailMap(Long.valueOf(since));
}
}
rv = new TreeMap<Long, String>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(_file), "UTF-8"));
String line = null;
while ((line = br.readLine()) != null) {
try {
String[] s = line.split(" ", 3);
if (!s[1].equals(event)) continue;
long time = Long.parseLong(s[0]);
if (time <= since) continue;
Long ltime = Long.valueOf(time);
String info = s.length > 2 ? s[2] : "";
rv.put(ltime, info);
} catch (IndexOutOfBoundsException ioobe) {
} catch (NumberFormatException nfe) {
}
}
rv = Collections.unmodifiableSortedMap(rv);
_cache.put(event, rv);
_cacheTime.put(event, Long.valueOf(since));
} catch (IOException ioe) {
} finally {
if (br != null)
try {
br.close();
} catch (IOException ioe) {
}
}
return rv;
}
@Override
public boolean isCriteriaViolated() {
if (memoryLog.isEmpty()
|| memoryLog.firstKey()
> System.currentTimeMillis()
- SmartRestart.getInstance().getConf().getTickSamplePeriod()) {
return false;
}
float val = 0;
SortedMap<Long, Float> subset =
memoryLog.tailMap(
System.currentTimeMillis()
- SmartRestart.getInstance().getConf().getMemorySamplePeriod());
for (Float f : subset.values()) {
val += f;
}
val /= subset.size();
return MemoryInterface.getUsedMemoryPercent() >= val;
}
private <K, V> void verify(
SortedMap<K, V> immutableMap, SortedMap<K, V> mutableMap, K key1, K key2, V value) {
try {
immutableMap.clear();
Assert.assertTrue(mutableMap.isEmpty());
} catch (UnsupportedOperationException e) {
Assert.assertFalse(mutableMap.isEmpty());
}
Assert.assertEquals(immutableMap.containsKey(key1), mutableMap.containsKey(key1));
Assert.assertEquals(immutableMap.containsKey(key2), mutableMap.containsKey(key2));
Assert.assertEquals(immutableMap.containsValue(value), mutableMap.containsValue(value));
Assert.assertEquals(immutableMap.containsValue(null), mutableMap.containsValue(null));
Assert.assertEquals(immutableMap.entrySet(), mutableMap.entrySet());
Assert.assertEquals(immutableMap.get(key1), mutableMap.get(key1));
Assert.assertEquals(immutableMap.get(key2), mutableMap.get(key2));
Assert.assertEquals(immutableMap.isEmpty(), mutableMap.isEmpty());
Assert.assertEquals(immutableMap.keySet(), mutableMap.keySet());
try {
immutableMap.put(key1, value);
Assert.fail();
} catch (UnsupportedOperationException e) {
}
try {
immutableMap.putAll(java.util.Collections.singletonMap(key1, value));
Assert.fail();
} catch (UnsupportedOperationException e) {
}
try {
immutableMap.remove(key1);
} catch (UnsupportedOperationException e) {
}
Assert.assertEquals(immutableMap.size(), mutableMap.size());
// Is it OK that this fails?
// Assert.assertEquals(immutableMap.values(), mutableMap.values());
Assert.assertEquals(immutableMap.values().size(), mutableMap.values().size());
if (!mutableMap.isEmpty()) {
Assert.assertEquals(
immutableMap.values().iterator().next(), mutableMap.values().iterator().next());
}
Assert.assertSame(immutableMap.comparator(), mutableMap.comparator());
if (!mutableMap.isEmpty()) {
Assert.assertEquals(immutableMap.firstKey(), mutableMap.firstKey());
Assert.assertEquals(immutableMap.lastKey(), mutableMap.lastKey());
} else {
try {
immutableMap.firstKey();
Assert.fail();
} catch (NoSuchElementException e) {
}
try {
immutableMap.lastKey();
Assert.fail();
} catch (NoSuchElementException e) {
}
}
Assert.assertEquals(immutableMap.headMap(key1), mutableMap.headMap(key1));
Assert.assertEquals(immutableMap.headMap(key2), mutableMap.headMap(key2));
Assert.assertEquals(immutableMap.subMap(key1, key2), mutableMap.subMap(key1, key2));
Assert.assertEquals(immutableMap.tailMap(key1), mutableMap.tailMap(key1));
Assert.assertEquals(immutableMap.tailMap(key2), mutableMap.tailMap(key2));
}
private <K, V> void checkOpenRanges(
Transaction txn,
int i,
EntityIndex<K, V> index,
SortedMap<Integer, SortedSet<Integer>> expected,
Getter<K> kGetter,
Getter<V> vGetter)
throws DatabaseException {
SortedMap<K, V> map = index.sortedMap();
SortedMap<Integer, SortedSet<Integer>> rangeExpected;
K k = kGetter.fromInt(i);
K kPlusOne = kGetter.fromInt(i + 1);
/* Head range exclusive. */
rangeExpected = expected.headMap(i);
checkCursor(
index.keys(txn, null, false, k, false, null),
map.headMap(k).keySet(),
true,
expandKeys(rangeExpected),
kGetter);
checkCursor(
index.entities(txn, null, false, k, false, null),
map.headMap(k).values(),
false,
expandValues(rangeExpected),
vGetter);
/* Head range inclusive. */
rangeExpected = expected.headMap(i + 1);
checkCursor(
index.keys(txn, null, false, k, true, null),
map.headMap(kPlusOne).keySet(),
true,
expandKeys(rangeExpected),
kGetter);
checkCursor(
index.entities(txn, null, false, k, true, null),
map.headMap(kPlusOne).values(),
false,
expandValues(rangeExpected),
vGetter);
/* Tail range exclusive. */
rangeExpected = expected.tailMap(i + 1);
checkCursor(
index.keys(txn, k, false, null, false, null),
map.tailMap(kPlusOne).keySet(),
true,
expandKeys(rangeExpected),
kGetter);
checkCursor(
index.entities(txn, k, false, null, false, null),
map.tailMap(kPlusOne).values(),
false,
expandValues(rangeExpected),
vGetter);
/* Tail range inclusive. */
rangeExpected = expected.tailMap(i);
checkCursor(
index.keys(txn, k, true, null, false, null),
map.tailMap(k).keySet(),
true,
expandKeys(rangeExpected),
kGetter);
checkCursor(
index.entities(txn, k, true, null, false, null),
map.tailMap(k).values(),
false,
expandValues(rangeExpected),
vGetter);
}
public TuningResDTO buildTuningRes(
Date startDate,
Date endDate,
Stock stock,
String analyseName,
SortedMap<Date, double[]> calcOutput,
EventInfo evtDef,
Observer observer,
Boolean isEventsPersisted)
throws NotEnoughDataException {
if (calcOutput == null) calcOutput = new TreeMap<Date, double[]>();
if (!calcOutput.isEmpty() && calcOutput.firstKey().before(startDate))
calcOutput = calcOutput.tailMap(startDate);
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy MM dd");
String noResMsg =
"No estimate is available for "
+ stock.getName()
+ " between "
+ dateFormat.format(startDate)
+ " and "
+ dateFormat.format(endDate)
+ " with "
+ evtDef
+ ".\n";
try {
Date endCalcRes =
(calcOutput.size() > 0 && calcOutput.lastKey().after(endDate))
? calcOutput.lastKey()
: endDate;
// Grab calculated events
HashSet<EventInfo> eventDefinitions = new HashSet<EventInfo>();
eventDefinitions.add(evtDef);
SymbolEvents eventsCalculated =
EventsResources.getInstance()
.crudReadEventsForStock(
stock, startDate, endCalcRes, isEventsPersisted, eventDefinitions, analyseName);
// Init event def list
NavigableSet<EventValue> eventListForEvtDef =
new TreeSet<EventValue>(
new Comparator<EventValue>() {
@Override
public int compare(EventValue o1, EventValue o2) {
return o1.getDate().compareTo(o2.getDate());
}
});
eventListForEvtDef.addAll(eventsCalculated.getDataResultMap().values());
return buildTuningRes(
stock,
startDate,
endDate,
endCalcRes,
analyseName,
calcOutput,
eventListForEvtDef,
noResMsg,
evtDef.info(),
observer);
} catch (NoQuotationsException e) {
LOGGER.warn(noResMsg, e);
throw new NotEnoughDataException(stock, noResMsg, e);
} catch (NotEnoughDataException e) {
LOGGER.warn(noResMsg, e);
throw e;
} catch (Exception e) {
LOGGER.error(noResMsg, e);
throw new NotEnoughDataException(stock, noResMsg, e);
}
}
private void updateChartIndicator(
final Stock selectedShare, boolean recalculationGranted, boolean needsUpdate) {
SortedMap<Date, double[]> outputCache =
chartTarget
.getHightlitedEventModel()
.getOutputCache(selectedShare, chartTarget.getChartedEvtDef());
if (outputCache == null || outputCache.isEmpty()) {
// No indic found despite recalc
if (!recalculationGranted && !needsUpdate) {
String errMsg =
"No output data are available for display within the period you have selected share "
+ selectedShare.getFriendlyName()
+ " and "
+ chartTarget.getChartedEvtDef().getEventReadableDef()
+ ".\n"
+ "If you just cleared the calculations results, you may want to Force and Update calculations.";
String addMsg =
"This may also happen if calculations failed, or if there is not enough quotations for the period.\n"
+
// "Check the calculators in "+TRENDBUTTXT+" as well as the date boundaries against
// the available quotations.\n" +
"You may want to check the date boundaries against the available quotations.\n"
+ "Also note that some calculators need full OLHC and Volume in order to be calculated.\n"
+ "If '"
+ chartTarget.getChartedEvtDef().getEventReadableDef()
+ "' is a calculator created by you, you may also want to check the formula.";
showPopupDialog(errMsg, "Ok", addMsg, null);
}
// Thats all good, we display
} else {
SortedMap<Date, double[]> subMap = new TreeMap<Date, double[]>();
if (!outputCache.isEmpty()) {
Calendar instance = Calendar.getInstance();
instance.setTime(chartTarget.getSlidingEndDate());
QuotationsFactories.getFactory().incrementDate(instance, 1);
Date endPlus1 = instance.getTime();
if (endPlus1.after(outputCache.lastKey())) {
subMap = outputCache.tailMap(this.chartTarget.getSlidingStartDate());
} else {
subMap = outputCache.subMap(this.chartTarget.getSlidingStartDate(), endPlus1);
}
if (!subMap.isEmpty())
chartTarget
.getMainChartWraper()
.updateIndicDataSet(
chartTarget.getChartedEvtDef(), subMap, chartTarget.getPlotChartDimensions());
}
if (chartSettingsPopup != null && !chartSettingsPopup.getSelectionShell().isDisposed()) {
initChartSettingsPopup(false);
}
}
}
public static void main(String[] args) throws Exception {
System.out.println(new Date() + " Starting.");
/* Iterate over extra-info descriptors to learn about bandwidth
* histories. Append 15-minute intervals of written bytes to
* status/written-bytes/$fingerprint. */
System.out.println(new Date() + " Reading in/extra-infos/* ...");
DescriptorReader extraInfoReader = DescriptorSourceFactory.createDescriptorReader();
extraInfoReader.addDirectory(new File("in/extra-infos"));
extraInfoReader.setExcludeFiles(new File("status/extra-info-history"));
Iterator<DescriptorFile> extraInfoFiles = extraInfoReader.readDescriptors();
while (extraInfoFiles.hasNext()) {
DescriptorFile extraInfoFile = extraInfoFiles.next();
if (extraInfoFile.getDescriptors() != null) {
for (Descriptor descriptor : extraInfoFile.getDescriptors()) {
ExtraInfoDescriptor extraInfoDescriptor = (ExtraInfoDescriptor) descriptor;
BandwidthHistory writeHistory = extraInfoDescriptor.getWriteHistory();
if (writeHistory == null) {
continue;
}
String fingerprint = extraInfoDescriptor.getFingerprint();
File writtenBytesFile = new File("status/written-bytes/" + fingerprint);
writtenBytesFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(writtenBytesFile, true));
for (Map.Entry<Long, Long> e : writeHistory.getBandwidthValues().entrySet()) {
long intervalEndMillis = e.getKey();
long bytesWritten = e.getValue();
bw.write(String.valueOf(intervalEndMillis) + " " + String.valueOf(bytesWritten) + "\n");
}
bw.close();
}
}
}
/* Iterate over exit lists to learn about exit IP addresses. Append
* lines to status/exit-addresses/$fingerprint. */
System.out.println(new Date() + " Reading in/exit-lists/* ...");
DescriptorReader exitListReader = DescriptorSourceFactory.createDescriptorReader();
exitListReader.addDirectory(new File("in/exit-lists"));
exitListReader.setExcludeFiles(new File("status/exit-list-history"));
Iterator<DescriptorFile> exitListFiles = exitListReader.readDescriptors();
while (exitListFiles.hasNext()) {
DescriptorFile exitListFile = exitListFiles.next();
if (exitListFile.getDescriptors() != null) {
for (Descriptor descriptor : exitListFile.getDescriptors()) {
ExitList exitList = (ExitList) descriptor;
if (exitList.getExitListEntries() == null) {
continue;
}
for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
String fingerprint = exitListEntry.getFingerprint();
File exitAddressesFile = new File("status/exit-addresses/" + fingerprint);
exitAddressesFile.getParentFile().mkdirs();
long scanMillis = exitListEntry.getScanMillis();
String address = exitListEntry.getExitAddress();
BufferedWriter bw = new BufferedWriter(new FileWriter(exitAddressesFile, true));
bw.write(String.valueOf(scanMillis) + " " + address + "\n");
bw.close();
}
}
}
}
/* Iterate over consensuses to learn about OR addresses of relays with
* the Exit flag. Append lines to
* status/or-addresses/$fingerprint. */
System.out.println(new Date() + " Reading in/consensuses/* ...");
DescriptorReader consensusReader = DescriptorSourceFactory.createDescriptorReader();
consensusReader.addDirectory(new File("in/consensuses"));
consensusReader.setExcludeFiles(new File("status/consensus-history"));
Iterator<DescriptorFile> consensusFiles = consensusReader.readDescriptors();
while (consensusFiles.hasNext()) {
DescriptorFile consensusFile = consensusFiles.next();
if (consensusFile.getDescriptors() != null) {
for (Descriptor descriptor : consensusFile.getDescriptors()) {
RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor;
if (consensus.getStatusEntries() == null) {
continue;
}
long validAfterMillis = consensus.getValidAfterMillis();
for (NetworkStatusEntry statusEntry : consensus.getStatusEntries().values()) {
if (!statusEntry.getFlags().contains("Exit")) {
continue;
}
String fingerprint = statusEntry.getFingerprint();
File orAddressesFile = new File("status/or-addresses/" + fingerprint);
orAddressesFile.getParentFile().mkdirs();
String address = statusEntry.getAddress();
BufferedWriter bw = new BufferedWriter(new FileWriter(orAddressesFile, true));
bw.write(String.valueOf(validAfterMillis) + " " + address + "\n");
bw.close();
}
}
}
}
/* Make sure not to overwrite existing results, and prepare writing
* results otherwise. */
File differentExitAddressFile = new File("out/different-exit-address.csv");
if (differentExitAddressFile.exists()) {
return;
} else {
differentExitAddressFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(differentExitAddressFile));
bw.write("timestamp,differentaddress,writtenbytes\n");
bw.close();
}
/* Iterate over OR addresses of relays with the Exit flag. */
System.out.println(new Date() + " Writing " + "out/different-exit-address.csv ...");
for (File orAddressesFile : new File("status/or-addresses").listFiles()) {
String fingerprint = orAddressesFile.getName();
/* For every relay, read OR addresses, bandwidth histories, and
* exit addresses to memory. */
SortedMap<Long, String> orAddresses = new TreeMap<Long, String>();
SortedMap<Long, Long> writtenBytes = new TreeMap<Long, Long>();
SortedMap<Long, String> exitAddresses = new TreeMap<Long, String>();
String line;
BufferedReader br = new BufferedReader(new FileReader(orAddressesFile));
while ((line = br.readLine()) != null) {
String[] parts = line.split(" ");
long validAfterMillis = Long.parseLong(parts[0]);
String address = parts[1];
orAddresses.put(validAfterMillis, address);
}
br.close();
File writtenBytesFile = new File("status/written-bytes/" + fingerprint);
if (!writtenBytesFile.exists()) {
continue;
}
br = new BufferedReader(new FileReader(writtenBytesFile));
while ((line = br.readLine()) != null) {
String[] parts = line.split(" ");
long intervalEndMillis = Long.parseLong(parts[0]);
long bytes = Long.parseLong(parts[1]);
writtenBytes.put(intervalEndMillis, bytes);
}
br.close();
File exitAddressesFile = new File("status/exit-addresses/" + fingerprint);
if (exitAddressesFile.exists()) {
br = new BufferedReader(new FileReader(exitAddressesFile));
while ((line = br.readLine()) != null) {
String[] parts = line.split(" ");
long scanMillis = Long.parseLong(parts[0]);
String address = parts[1];
exitAddresses.put(scanMillis, address);
}
br.close();
}
/* Go through consensuses containing this relay as Exit relay in
* chronological order, sum up written bytes in the hour after the
* consensuses' valid-after time, and look up any exit addresses
* found up to 23 hours before up to 1 hour after the valid-after
* time. */
BufferedWriter bw = new BufferedWriter(new FileWriter(differentExitAddressFile, true));
for (Map.Entry<Long, String> e : orAddresses.entrySet()) {
long validAfterMillis = e.getKey();
String currentOrAddress = e.getValue();
long currentWrittenBytes = 0L;
for (long currentBytes :
writtenBytes
.tailMap(validAfterMillis)
.headMap(validAfterMillis + 1L * 60L * 60L * 1000L)
.values()) {
currentWrittenBytes += currentBytes;
}
if (currentWrittenBytes < 1L) {
continue;
}
Set<String> currentExitAddresses = new HashSet<String>();
for (String currentExitAddress :
exitAddresses
.tailMap(validAfterMillis - 23L * 60L * 60L * 1000L)
.headMap(validAfterMillis + 1L * 60L * 60L * 1000L)
.values()) {
if (!currentExitAddress.equals(currentOrAddress)) {
currentExitAddresses.add(currentExitAddress);
}
}
boolean usedOtherAddress = !currentExitAddresses.isEmpty();
bw.write(
String.valueOf(validAfterMillis / 1000L)
+ ","
+ (usedOtherAddress ? "TRUE" : "FALSE")
+ ","
+ currentWrittenBytes
+ "\n");
}
bw.close();
}
System.out.println(new Date() + " Terminating.");
}
/**
* @see net.sf.hajdbc.invocation.InvocationStrategy#invoke(net.sf.hajdbc.sql.SQLProxy,
* net.sf.hajdbc.invocation.Invoker)
*/
@Override
public <Z, D extends Database<Z>, T, R, E extends Exception> SortedMap<D, R> invoke(
SQLProxy<Z, D, T, E> proxy, Invoker<Z, D, T, R, E> invoker) throws E {
Map.Entry<SortedMap<D, R>, SortedMap<D, E>> results = this.collectResults(proxy, invoker);
SortedMap<D, R> resultMap = results.getKey();
SortedMap<D, E> exceptionMap = results.getValue();
if (!exceptionMap.isEmpty()) {
ExceptionFactory<E> exceptionFactory = proxy.getExceptionFactory();
DatabaseCluster<Z, D> cluster = proxy.getDatabaseCluster();
Dialect dialect = cluster.getDialect();
List<D> failedDatabases = new ArrayList<D>(exceptionMap.size());
// Determine which exceptions are due to failures
for (Map.Entry<D, E> entry : exceptionMap.entrySet()) {
if (exceptionFactory.indicatesFailure(entry.getValue(), dialect)) {
failedDatabases.add(entry.getKey());
}
}
StateManager stateManager = cluster.getStateManager();
// Deactivate failed databases, unless all failed
if (!resultMap.isEmpty() || (failedDatabases.size() < exceptionMap.size())) {
for (D failedDatabase : failedDatabases) {
E exception = exceptionMap.remove(failedDatabase);
if (cluster.deactivate(failedDatabase, stateManager)) {
logger.error(
"" + exception + "" + Messages.DATABASE_DEACTIVATED.getMessage(),
failedDatabase,
cluster);
}
}
}
if (!exceptionMap.isEmpty()) {
// If primary database threw exception
if (resultMap.isEmpty() || !exceptionMap.headMap(resultMap.firstKey()).isEmpty()) {
D primaryDatabase = exceptionMap.firstKey();
E primaryException = exceptionMap.get(primaryDatabase);
// Deactivate databases with non-matching exceptions
for (Map.Entry<D, E> entry : exceptionMap.tailMap(primaryDatabase).entrySet()) {
E exception = entry.getValue();
if (!exceptionFactory.equals(exception, primaryException)) {
D database = entry.getKey();
if (cluster.deactivate(database, stateManager)) {
logger.error("" + exception + " " + Messages.DATABASE_INCONSISTENT.getMessage());
}
}
}
// Deactivate databases with results
for (Map.Entry<D, R> entry : resultMap.entrySet()) {
D database = entry.getKey();
if (cluster.deactivate(database, stateManager)) {
logger.error(Messages.DATABASE_INCONSISTENT.getMessage());
}
}
throw primaryException;
}
}
// Else primary was successful
// Deactivate databases with exceptions
for (Map.Entry<D, E> entry : exceptionMap.entrySet()) {
D database = entry.getKey();
E exception = entry.getValue();
if (cluster.deactivate(database, stateManager)) {
logger.error(Messages.DATABASE_DEACTIVATED.getMessage(), database, cluster);
}
}
}
return resultMap;
}
public SortedMap<K, V> tailMap(K k) {
checkInit();
return delegate.tailMap(k);
}
