public static void readTest(String tableStr, String row) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] rowkey = Bytes.toBytes(row);
Get get = new Get(rowkey);
get.addFamily(f0);
Result result = table.get(get);
NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(qualifier + ":" + value);
}
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
public String toString() {
if (variant == null
|| variant.isEmpty()
|| destinationName == null
|| destinationName.isEmpty()) {
return "jms://";
}
final StringBuilder buf = new StringBuilder("jms:");
buf.append(variant).append(':').append(encode(destinationName));
if (!parameters.isEmpty()) {
buf.append('?');
boolean addDelimiter = false;
for (Entry<String, String> param : parameters.entrySet()) {
if (addDelimiter) {
buf.append('&');
} else {
addDelimiter = true;
}
buf.append(encode(param.getKey()));
final String value = param.getValue();
if (value != null && value.length() > 0) {
buf.append('=').append(encode(value));
}
}
}
return buf.toString();
}
@Override
public boolean rollbackTx() throws Exception {
if (consumingEntries.isEmpty()) {
return true;
}
// Put the consuming entries back to cache
entryCache.putAll(consumingEntries);
// If not committed, no need to update HBase.
if (!committed) {
return true;
}
commitCount -= consumingEntries.size();
// Revert changes in HBase rows
// If it is FIFO, restore to the CLAIMED state. This instance will retry it on the next dequeue.
if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
&& consumerConfig.getGroupSize() > 1) {
byte[] stateContent = encodeStateColumn(ConsumerEntryState.CLAIMED);
updateState(consumingEntries.keySet(), stateColumnName, stateContent);
} else {
undoState(consumingEntries.keySet(), stateColumnName);
}
return true;
}
public Word getRandomWord() {
double value = random.nextDouble() * total;
if (weightsMap.isEmpty()) {
return new Word(null, 1.0);
}
return weightsMap.ceilingEntry(value).getValue();
}
private int flushRing(int writeGroupsToFlush, int flushedGroups, boolean forceFlush)
throws IOException {
NavigableMap<GroupKey, WriteGroup> subMap = writeGroups.tailMap(lastGroupKey, false);
if (!subMap.isEmpty()) {
flushedGroups = iterateBySubRing(subMap, writeGroupsToFlush, 0, forceFlush);
if (flushedGroups < writeGroupsToFlush) {
if (!subMap.isEmpty()) {
subMap = writeGroups.headMap(subMap.firstKey(), false);
flushedGroups = iterateBySubRing(subMap, writeGroupsToFlush, flushedGroups, forceFlush);
}
}
} else
flushedGroups =
iterateBySubRing(writeGroups, writeGroupsToFlush, flushedGroups, forceFlush);
return flushedGroups;
}
public Word getWord(int position) throws Exception {
if (!weightsMap.isEmpty() && position >= 0 && position < weightsMap.size()) {
int count = 0;
for (Entry<Double, Word> entry : weightsMap.entrySet()) {
if (count == position) return entry.getValue();
count++;
}
}
throw new Exception("Wrong position \"" + position + "\" for dictionary \"" + name + "\".");
}
@Override
public boolean commitTx() throws Exception {
if (consumingEntries.isEmpty()) {
return true;
}
byte[] stateContent = encodeStateColumn(ConsumerEntryState.PROCESSED);
updateState(consumingEntries.keySet(), stateColumnName, stateContent);
commitCount += consumingEntries.size();
committed = true;
return true;
}
private NavigableMap<Number640, Data> getLatestInternalOrig(NavigableMap<Number640, Data> input) {
// delete all predecessors
NavigableMap<Number640, Data> result = new TreeMap<Number640, Data>();
while (!input.isEmpty()) {
// first entry is a latest version
Entry<Number640, Data> latest = input.lastEntry();
// store in results list
result.put(latest.getKey(), latest.getValue());
// delete all predecessors of latest entry
deletePredecessors(latest.getKey(), input);
}
return result;
}
@Override
protected ApacheLogEntry prefetch() throws Exception {
while (sortedBuffer.size() < maxSortedBufferSize) {
if (itEntry == null) {
if (itFile.hasNext()) {
itEntry = new ApacheLogEntryIterator(LogDirectory.open(itFile.next().getValue()), true);
} else {
break;
}
}
if (!itEntry.hasNext()) {
itEntry = null;
continue;
}
ApacheLogEntry entry = itEntry.next();
if (low != null) {
int d = entry.getDate().compareTo(low);
if (d < 0 || (d == 0 && !lowInclusive)) {
continue;
}
}
sortedBuffer.put(entry.getDate(), entry);
}
if (!sortedBuffer.isEmpty()) {
ApacheLogEntry entry = sortedBuffer.pollFirstEntry().getValue();
if (sanityCheckMonotonictyDate != null) {
if (sanityCheckMonotonictyDate.compareTo(entry.getDate()) > 0) {
throw new RuntimeException("Dates are not monoton");
}
}
sanityCheckMonotonictyDate = entry.getDate();
if (high != null) {
int d = high.compareTo(entry.getDate());
if (d < 0 || (d == 0 && !highInclusive)) {
return finish();
}
}
return entry;
}
return finish();
}
@Override
public DequeueResult<byte[]> dequeue(int maxBatchSize) throws IOException {
Preconditions.checkArgument(maxBatchSize > 0, "Batch size must be > 0.");
// pre-compute the "claimed" state content in case of FIFO.
byte[] claimedStateValue = null;
if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
&& consumerConfig.getGroupSize() > 1) {
claimedStateValue = encodeStateColumn(ConsumerEntryState.CLAIMED);
}
while (consumingEntries.size() < maxBatchSize && getEntries(consumingEntries, maxBatchSize)) {
// ANDREAS: this while loop should stop once getEntries/populateCache reaches the end of the
// queue. Currently, it
// will retry as long as it gets at least one entry in every round, even if that is an entry
// that must be ignored
// because it cannot be claimed.
// ANDREAS: It could be a problem that we always read to the end of the queue. This way one
// flowlet instance may
// always all entries, while others are idle.
// For FIFO, need to try claiming the entry if group size > 1
if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
&& consumerConfig.getGroupSize() > 1) {
Iterator<Map.Entry<byte[], SimpleQueueEntry>> iterator =
consumingEntries.entrySet().iterator();
while (iterator.hasNext()) {
SimpleQueueEntry entry = iterator.next().getValue();
if (entry.getState() == null
|| QueueEntryRow.getStateInstanceId(entry.getState())
>= consumerConfig.getGroupSize()) {
// If not able to claim it, remove it, and move to next one.
if (!claimEntry(entry.getRowKey(), claimedStateValue)) {
iterator.remove();
}
}
}
}
}
// If nothing get dequeued, return the empty result.
if (consumingEntries.isEmpty()) {
return EMPTY_RESULT;
}
return new SimpleDequeueResult(consumingEntries.values());
}
@Override
public void postTxCommit() {
if (scanStartRow != null) {
if (!consumingEntries.isEmpty()) {
// Start row can be updated to the largest rowKey in the consumingEntries (now is consumed)
// that is smaller than or equal to scanStartRow
byte[] floorKey = consumingEntries.floorKey(scanStartRow);
if (floorKey != null) {
startRow = floorKey;
}
} else {
// If the dequeue has empty result, startRow can advance to scanStartRow
startRow = Arrays.copyOf(scanStartRow, scanStartRow.length);
}
}
}
public static void scanTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_0");
byte[] stopRow = Bytes.toBytes("rowKey_6");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
while (result != null) {
String rowKey = Bytes.toString(result.getRow());
NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(rowKey + ":" + qualifier + ":" + value);
}
result = scanner.next();
count++;
System.out.println("-----------------------------");
}
table.close(); // very important
System.out.println("count:" + count);
} catch (IOException e) {
e.printStackTrace();
}
}
private static List<ExecutionScript> loadScripts(
String flowId, ExecutionPhase phase, NavigableMap<String, String> contents) {
assert flowId != null;
assert phase != null;
assert contents != null;
if (contents.isEmpty()) {
return Collections.emptyList();
}
List<ExecutionScript> results = new ArrayList<ExecutionScript>();
Map<String, NavigableMap<String, String>> scripts = partitioning(contents);
for (Map.Entry<String, NavigableMap<String, String>> entry : scripts.entrySet()) {
String scriptId = entry.getKey();
NavigableMap<String, String> scriptContents = entry.getValue();
ExecutionScript script = loadScript(flowId, phase, scriptId, scriptContents);
results.add(script);
}
checkBlockers(flowId, phase, results);
return results;
}
@Override
protected ApacheLogEntry prefetch() throws Exception {
String line;
while (sortedBuffer.size() < maxSortedBufferSize && ((line = reader.readLine()) != null)) {
try {
ApacheLogEntry entry = ApacheLogEntry.parse(line);
sortedBuffer.put(entry.getDate(), entry);
} catch (ParseException e) {
// TODO Shouldn't happen, but the parser is sucky, so it happens
e.printStackTrace();
}
}
if (!sortedBuffer.isEmpty()) {
return sortedBuffer.pollFirstEntry().getValue();
}
return finish();
}
public Word getNextWord(String previousWord, boolean limit) throws Exception {
if (weightsMap.isEmpty()) {
return new Word(null, 1.0);
}
if (null == previousWord || previousWord.equals(weightsMap.lastEntry().getValue().getValue())) {
if (!limit) {
return weightsMap.firstEntry().getValue();
}
throw new LimitReachedException(
previousWord, "Last word in dictionary \"" + name + "\" already reached.");
}
Iterator<Word> iterator = weightsMap.values().iterator();
while (iterator.hasNext()) {
if (previousWord.equals(iterator.next().getValue())) {
return iterator.next();
}
}
throw new Exception(
"No previous word \"" + previousWord + "\" was found in dictionary \"" + name + "\".");
}
private static ExecutionScript loadScript(
String flowId, ExecutionPhase phase, String nodeId, Map<String, String> contents) {
assert flowId != null;
assert phase != null;
assert nodeId != null;
assert contents != null;
String prefix = getPrefix(flowId, phase, nodeId);
String scriptId = extract(contents, prefix, KEY_ID);
String kindSymbol = extract(contents, prefix, KEY_KIND);
ExecutionScript.Kind kind = ExecutionScript.Kind.findFromSymbol(kindSymbol);
String blockersString = extract(contents, prefix, KEY_BLOCKERS);
Set<String> blockers = parseTokens(blockersString);
Map<String, String> environmentVariables =
PropertiesUtil.createPrefixMap(contents, KEY_ENV_PREFIX);
ExecutionScript script;
if (kind == ExecutionScript.Kind.COMMAND) {
String profileName = extract(contents, prefix, KEY_PROFILE);
String moduleName = extract(contents, prefix, KEY_MODULE);
NavigableMap<String, String> commandMap =
PropertiesUtil.createPrefixMap(contents, KEY_COMMAND_PREFIX);
if (commandMap.isEmpty()) {
throw new IllegalArgumentException(
MessageFormat.format("\"{0}*\" is not defined", prefix + KEY_COMMAND_PREFIX));
}
List<String> command = new ArrayList<String>(commandMap.values());
script =
new CommandScript(
scriptId, blockers, profileName, moduleName, command, environmentVariables);
} else if (kind == ExecutionScript.Kind.HADOOP) {
String className = extract(contents, prefix, KEY_CLASS_NAME);
Map<String, String> properties = PropertiesUtil.createPrefixMap(contents, KEY_PROP_PREFIX);
script = new HadoopScript(scriptId, blockers, className, properties, environmentVariables);
} else {
throw new IllegalArgumentException(
MessageFormat.format("Unsupported kind in \"{0}\": {1}", prefix + KEY_KIND, kindSymbol));
}
LOG.trace("Loaded script {}* -> {}", script);
return script;
}
private static Map<String, NavigableMap<String, String>> partitioning(
NavigableMap<String, String> map) {
assert map != null;
Map<String, NavigableMap<String, String>> results =
new TreeMap<String, NavigableMap<String, String>>();
while (map.isEmpty() == false) {
String name = map.firstKey();
int index = name.indexOf('.');
if (index >= 0) {
name = name.substring(0, index);
}
String first = name + '.';
String last = name + (char) ('.' + 1);
NavigableMap<String, String> partition = new TreeMap<String, String>();
for (Map.Entry<String, String> entry : map.subMap(first, last).entrySet()) {
String key = entry.getKey();
partition.put(key.substring(name.length() + 1), entry.getValue());
}
results.put(name, partition);
map.remove(name);
map.subMap(first, last).clear();
}
return results;
}
@Override
public boolean isEmpty() {
return configMap.isEmpty();
}
/**
* HBaseの一行をItemDataに変換
*
* @param r row in HBase
* @return 商品情報
*/
public ItemData toDataStore(Result r) {
if (r == null || r.isEmpty()) {
return null;
}
ItemData item = null;
// meta family
NavigableMap<byte[], byte[]> map = r.getFamilyMap(META_FAMILY);
if (map != null && !map.isEmpty()) {
item = new ItemData();
// ctime
byte[] bCtime = map.get(Bytes.toBytes("ctime"));
if (bCtime != null) {
item.ctime = Bytes.toLong(bCtime);
}
}
// data family
map = r.getFamilyMap(DATA_FAMILY);
if (map != null && !map.isEmpty()) {
if (item == null) {
item = new ItemData();
}
// shop ID
byte[] bShopID = map.get(Bytes.toBytes("shop_id"));
if (bShopID != null) {
item.shopID = Bytes.toInt(bShopID);
}
// item ID
byte[] bItemID = map.get(Bytes.toBytes("item_id"));
if (bItemID != null) {
item.itemID = Bytes.toInt(bItemID);
}
// genre ID
byte[] bGenreID = map.get(Bytes.toBytes("genre_id"));
if (bGenreID != null) {
item.genreID = Bytes.toInt(bGenreID);
}
// item name
byte[] bItemName = map.get(Bytes.toBytes("item_name"));
if (bItemName != null) {
item.itemName = Bytes.toString(bItemName);
}
// full item url
byte[] bFullItemUrl = map.get(Bytes.toBytes("full_item_url"));
if (bFullItemUrl != null) {
item.fullItemUrl = Bytes.toString(bFullItemUrl);
}
// price
byte[] bPrice = map.get(Bytes.toBytes("price"));
if (bPrice != null) {
item.price = Bytes.toString(bPrice);
}
}
if (item == null) {
item = new ItemData();
}
item.rowkey = r.getRow();
return item;
}
/**
* Sends all messages currently in forward_table to the new coordinator (changing the dest field).
* This needs to be done, so the underlying reliable unicast protocol (e.g. UNICAST) adds these
* messages to its retransmission mechanism<br>
* Note that we need to resend the messages in order of their seqnos ! We also need to prevent
* other message from being inserted until we're done, that's why there's synchronization.<br>
* Access to the forward_table doesn't need to be synchronized as there won't be any insertions
* during flushing (all down-threads are blocked)
*/
protected void flushMessagesInForwardTable() {
if (is_coord) {
for (Map.Entry<Long, Message> entry : forward_table.entrySet()) {
Long key = entry.getKey();
Message msg = entry.getValue();
Buffer buf;
try {
buf = Util.streamableToBuffer(msg);
} catch (Exception e) {
log.error(Util.getMessage("FlushingBroadcastingFailed"), e);
continue;
}
SequencerHeader hdr = new SequencerHeader(SequencerHeader.WRAPPED_BCAST, key);
Message forward_msg = new Message(null, buf).putHeader(this.id, hdr);
if (log.isTraceEnabled())
log.trace(local_addr + ": flushing (broadcasting) " + local_addr + "::" + key);
down_prot.down(new Event(Event.MSG, forward_msg));
}
return;
}
// for forwarded messages, we need to receive the forwarded message from the coordinator, to
// prevent this case:
// - V1={A,B,C}
// - A crashes
// - C installs V2={B,C}
// - C forwards messages 3 and 4 to B (the new coord)
// - B drops 3 because its view is still V1
// - B installs V2
// - B receives message 4 and broadcasts it
// ==> C's message 4 is delivered *before* message 3 !
// ==> By resending 3 until it is received, then resending 4 until it is received, we make sure
// this won't happen
// (see https://issues.jboss.org/browse/JGRP-1449)
while (flushing && running && !forward_table.isEmpty()) {
Map.Entry<Long, Message> entry = forward_table.firstEntry();
final Long key = entry.getKey();
Message msg = entry.getValue();
Buffer buf;
try {
buf = Util.streamableToBuffer(msg);
} catch (Exception e) {
log.error(Util.getMessage("FlushingBroadcastingFailed"), e);
continue;
}
while (flushing && running && !forward_table.isEmpty()) {
SequencerHeader hdr = new SequencerHeader(SequencerHeader.FLUSH, key);
Message forward_msg =
new Message(coord, buf).putHeader(this.id, hdr).setFlag(Message.Flag.DONT_BUNDLE);
if (log.isTraceEnabled())
log.trace(
local_addr
+ ": flushing (forwarding) "
+ local_addr
+ "::"
+ key
+ " to coord "
+ coord);
ack_promise.reset();
down_prot.down(new Event(Event.MSG, forward_msg));
Long ack = ack_promise.getResult(500);
if ((Objects.equals(ack, key)) || !forward_table.containsKey(key)) break;
}
}
}
boolean isEmpty() {
return rangeMap.isEmpty();
}
public Map<Number640, Enum<?>> putAll(
final NavigableMap<Number640, Data> dataMap,
PublicKey publicKey,
boolean putIfAbsent,
boolean domainProtection,
boolean sendSelf) {
if (dataMap.isEmpty()) {
return Collections.emptyMap();
}
final Number640 min = dataMap.firstKey();
final Number640 max = dataMap.lastKey();
final Map<Number640, Enum<?>> retVal = new HashMap<Number640, Enum<?>>();
final HashSet<Number480> keysToCheck = new HashSet<Number480>();
final RangeLock<Number640>.Range lock = lock(min, max);
try {
for (Map.Entry<Number640, Data> entry : dataMap.entrySet()) {
Number640 key = entry.getKey();
keysToCheck.add(key.locationAndDomainAndContentKey());
Data newData = entry.getValue();
if (!securityDomainCheck(
key.locationAndDomainKey(), publicKey, publicKey, domainProtection)) {
retVal.put(key, PutStatus.FAILED_SECURITY);
newData.release();
continue;
}
// We need this check in case we did not use the encoder/decoder,
// which is the case if we send the message to ourself. In that
// case, the public key of the data is never set to the message
// publick key, if the publick key of the data was null.
final PublicKey dataKey;
if (sendSelf && newData.publicKey() == null) {
dataKey = publicKey;
} else {
dataKey = newData.publicKey();
}
if (!securityEntryCheck(
key.locationAndDomainAndContentKey(), publicKey, dataKey, newData.isProtectedEntry())) {
retVal.put(key, PutStatus.FAILED_SECURITY);
newData.release();
continue;
}
final Data oldDataGet = backend.get(key);
if (oldDataGet != null) {
if (putIfAbsent) {
retVal.put(key, PutStatus.FAILED_NOT_ABSENT);
newData.release();
continue;
}
if (oldDataGet.isDeleted()) {
retVal.put(key, PutStatus.DELETED);
newData.release();
continue;
}
if (!oldDataGet.basedOnSet().equals(newData.basedOnSet())) {
retVal.put(key, PutStatus.VERSION_FORK);
newData.release();
continue;
}
}
final Data oldDataPut = backend.put(key, newData);
long expiration = newData.expirationMillis();
// handle timeout
backend.addTimeout(key, expiration);
if (newData.hasPrepareFlag()) {
retVal.put(key, PutStatus.OK_PREPARED);
} else {
retVal.put(key, PutStatus.OK);
}
if (oldDataPut != null && oldDataPut != newData) {
oldDataPut.release();
}
}
for (Number480 key : keysToCheck) {
// now check for forks
Number640 minVersion = new Number640(key, Number160.ZERO);
Number640 maxVersion = new Number640(key, Number160.MAX_VALUE);
NavigableMap<Number640, Data> tmp = backend.subMap(minVersion, maxVersion);
tmp = filterCopyOrig(tmp, -1, true);
NavigableMap<Number640, Data> heads = getLatestInternalOrig(tmp);
final boolean forked = heads.size() > 1;
for (final Map.Entry<Number640, Data> entry : heads.entrySet()) {
if (forked) {
if (retVal.containsKey(entry.getKey())) {
retVal.put(entry.getKey(), PutStatus.VERSION_FORK);
}
}
}
// now remove old versions
if (maxVersions > 0) {
NavigableMap<Number640, Data> versions = backend.subMap(minVersion, maxVersion);
while (!versions.isEmpty()
&& versions.firstKey().versionKey().timestamp() + maxVersions
<= versions.lastKey().versionKey().timestamp()) {
Map.Entry<Number640, Data> entry = versions.pollFirstEntry();
Data removed = backend.remove(entry.getKey(), true);
if (removed != null) {
removed.release();
}
backend.removeTimeout(entry.getKey());
}
}
}
return retVal;
} finally {
lock.unlock();
}
}
@MapFeature.Require(SUPPORTS_REMOVE)
@CollectionSize.Require(ONE)
public void testSingletonMapPollLast() {
assertEquals(a, navigableMap.pollLastEntry());
assertTrue(navigableMap.isEmpty());
}
public boolean isEmpty() {
return data.isEmpty();
}