public void remove(OIdentifiable identifiable) {
if (removeFromNewEntries(identifiable)) {
if (size >= 0) size--;
} else {
final Change counter = changes.get(identifiable);
if (counter == null) {
// Not persistent keys can only be in changes or newEntries
if (identifiable.getIdentity().isPersistent()) {
changes.put(identifiable, new DiffChange(-1));
size = -1;
} else
// Return immediately to prevent firing of event
return;
} else {
counter.decrement();
if (size >= 0)
if (counter.isUndefined()) size = -1;
else size--;
}
}
if (this.owner != null) ORecordInternal.unTrack(this.owner, identifiable);
if (updateOwner)
fireCollectionChangedEvent(
new OMultiValueChangeEvent<OIdentifiable, OIdentifiable>(
OMultiValueChangeEvent.OChangeType.REMOVE, identifiable, null, identifiable, false));
}
public void add(final OIdentifiable identifiable) {
if (identifiable == null)
throw new NullPointerException("Impossible to add a null identifiable in a ridbag");
if (identifiable.getIdentity().isValid()) {
Change counter = changes.get(identifiable);
if (counter == null) changes.put(identifiable, new DiffChange(1));
else {
if (counter.isUndefined()) {
counter = getAbsoluteValue(identifiable);
changes.put(identifiable, counter);
}
counter.increment();
}
} else {
final OModifiableInteger counter = newEntries.get(identifiable);
if (counter == null) newEntries.put(identifiable, new OModifiableInteger(1));
else counter.increment();
}
if (size >= 0) size++;
if (this.owner != null) ORecordInternal.track(this.owner, identifiable);
if (updateOwner)
fireCollectionChangedEvent(
new OMultiValueChangeEvent<OIdentifiable, OIdentifiable>(
OMultiValueChangeEvent.OChangeType.ADD, identifiable, identifiable, null, false));
}
@Override
public Set<Element> get(Object key) {
Get get = new Get(ByteArraySerializer.fromObject(key));
Result result;
try {
result = backingTable.get(get);
} catch (IOException e) {
LOG.severe("Cannot get from backing table");
e.printStackTrace();
return null;
}
NavigableMap<byte[], byte[]> map = result.getFamilyMap(Bytes.toBytes(VALUES));
if (null == map) return null;
HashSet<Element> elementHashSet = new HashSet<Element>();
for (byte[] byteArray : map.keySet()) {
if (indexClass.equals(HVertex.class) || indexClass.equals(Vertex.class)) {
HVertex hVertex = new HVertex(hGraph);
hVertex.setId(byteArray);
elementHashSet.add(hVertex);
} else {
final HEdge hEdge = new HEdge(hGraph);
hEdge.setId(byteArray);
elementHashSet.add(hEdge);
}
}
return elementHashSet;
}
@Override
public boolean convertRecords2Links() {
final Map<OIdentifiable, Change> newChangedValues = new HashMap<OIdentifiable, Change>();
for (Map.Entry<OIdentifiable, Change> entry : changes.entrySet()) {
OIdentifiable identifiable = entry.getKey();
if (identifiable instanceof ORecord) {
ORID identity = identifiable.getIdentity();
ORecord record = (ORecord) identifiable;
identity = record.getIdentity();
newChangedValues.put(identity, entry.getValue());
} else newChangedValues.put(entry.getKey().getIdentity(), entry.getValue());
}
for (Map.Entry<OIdentifiable, Change> entry : newChangedValues.entrySet()) {
if (entry.getKey() instanceof ORecord) {
ORecord record = (ORecord) entry.getKey();
newChangedValues.put(record, entry.getValue());
} else return false;
}
newEntries.clear();
changes.clear();
changes.putAll(newChangedValues);
return true;
}
@Override
public void setOwner(ORecord owner) {
if (owner != null && this.owner != null && !this.owner.equals(owner)) {
throw new IllegalStateException(
"This data structure is owned by document "
+ owner
+ " if you want to use it in other document create new rid bag instance and copy content of current one.");
}
if (this.owner != null) {
for (OIdentifiable entry : newEntries.keySet()) {
ORecordInternal.unTrack(this.owner, entry);
}
for (OIdentifiable entry : changes.keySet()) {
ORecordInternal.unTrack(this.owner, entry);
}
}
this.owner = owner;
if (this.owner != null) {
for (OIdentifiable entry : newEntries.keySet()) {
ORecordInternal.track(this.owner, entry);
}
for (OIdentifiable entry : changes.keySet()) {
ORecordInternal.track(this.owner, entry);
}
}
}
@Test
public void testHBColumnMultiVersion() {
Double[] testNumbers = new Double[] {3.14159, 2.71828, 0.0};
for (Double n : testNumbers) {
// Written as unversioned, read as versioned
Result result = hbMapper.writeValueAsResult(new CrawlNoVersion("key").setF1(n));
Crawl versioned = hbMapper.readValue(result, Crawl.class);
NavigableMap<Long, Double> columnHistory = versioned.getF1();
assertEquals("Column history size mismatch", 1, columnHistory.size());
assertEquals(
String.format(
"Inconsistency between %s and %s",
HBColumn.class.getSimpleName(), HBColumnMultiVersion.class.getSimpleName()),
n,
columnHistory.lastEntry().getValue());
// Written as versioned, read as unversioned
Result result1 =
hbMapper.writeValueAsResult(
new Crawl("key")
.addF1(Double.MAX_VALUE)
.addF1(Double.MAX_VALUE)
.addF1(Double.MAX_VALUE)
.addF1(n));
CrawlNoVersion unversioned = hbMapper.readValue(result1, CrawlNoVersion.class);
Double f1 = unversioned.getF1();
assertEquals(
String.format(
"Inconsistency between %s and %s",
HBColumnMultiVersion.class.getSimpleName(), HBColumn.class.getSimpleName()),
n,
f1);
}
}
public int run(String[] args) throws Exception {
Configuration argConf = getConf();
// JobConf conf = new JobConf(diffdb.class);
Configuration config = HBaseConfiguration.create();
HBaseAdmin hbAdmin = new HBaseAdmin(config);
dbutil db_util = new dbutil(config);
HTable runTable = new HTable(config, "gestore_runs");
Get runGet = new Get(argConf.get("id").getBytes());
Result pipeline = runTable.get(runGet);
NavigableMap<byte[], byte[]> pipeMap = pipeline.getFamilyMap("d".getBytes());
Map.Entry<byte[], byte[]> results = pipeMap.pollFirstEntry();
HashMap<String, HashMap<String, String>> resultMap =
new HashMap<String, HashMap<String, String>>();
while (results != null) {
String resultKey = new String(results.getKey());
String resultValue = new String(results.getValue());
String field = "type";
HashMap<String, String> tempMap = new HashMap<String, String>();
String entry = resultKey;
if (resultKey.endsWith("_db_timestamp")) {
field = "db_timestamp";
entry = resultKey.substring(0, resultKey.lastIndexOf("_db_timestamp"));
} else if (resultKey.endsWith("_filename")) {
field = "filename";
entry = resultKey.substring(0, resultKey.lastIndexOf("_filename"));
} else if (resultKey.endsWith("_regex")) {
field = "regex";
entry = resultKey.substring(0, resultKey.lastIndexOf("_regex"));
}
if (resultMap.containsKey(entry)) {
tempMap = resultMap.get(entry);
}
tempMap.put(field, resultValue);
resultMap.put(entry, tempMap);
// System.out.println("Key: " + resultKey + " Value: " + resultValue);
results = pipeMap.pollFirstEntry();
}
for (String key : resultMap.keySet()) {
System.out.println("File ID: " + key);
for (String subKey : resultMap.get(key).keySet()) {
// System.out.println("\t " + subKey + "\t\t" + resultMap.get(key).get(subKey));
System.out.format(" %1$-20s %2$s\n", subKey, resultMap.get(key).get(subKey));
}
}
return 0;
}
public IndexRateSource update(Collection<IndexRate> indexRates) {
for (IndexRate indexRate : indexRates) {
if (!data.containsKey(indexRate.indexId)) {
data.put(indexRate.indexId, new ConcurrentSkipListMap<>());
}
NavigableMap<LocalDate, IndexRate> dateMap = data.get(indexRate.indexId);
dateMap.put(indexRate.date, indexRate);
}
return this;
}
/**
* {@inheritDoc}
*
* @see java.util.NavigableSet#headSet(Object, boolean)
* @since 1.6
*/
@SuppressWarnings("unchecked")
public NavigableSet<E> headSet(E end, boolean endInclusive) {
// Check for errors
Comparator<? super E> c = backingMap.comparator();
if (c == null) {
((net.sourceforge.retroweaver.harmony.runtime.java.lang.Comparable<E>) end).compareTo(end);
} else {
c.compare(end, end);
}
return new TreeSet<E>(backingMap.headMap(end, endInclusive));
}
/**
* {@inheritDoc}
*
* @see java.util.NavigableSet#tailSet(Object, boolean)
* @since 1.6
*/
@SuppressWarnings("unchecked")
public NavigableSet<E> tailSet(E start, boolean startInclusive) {
// Check for errors
Comparator<? super E> c = backingMap.comparator();
if (c == null) {
((net.sourceforge.retroweaver.harmony.runtime.java.lang.Comparable<E>) start)
.compareTo(start);
} else {
c.compare(start, start);
}
return new TreeSet<E>(backingMap.tailMap(start, startInclusive));
}
/**
* Save the state of the {@code TreeSet} instance to a stream (that is, serialize it).
*
* @serialData Emits the comparator used to order this set, or {@code null} if it obeys its
* elements' natural ordering (Object), followed by the size of the set (the number of
* elements it contains) (int), followed by all of its elements (each an Object) in order (as
* determined by the set's Comparator, or by the elements' natural ordering if the set has no
* Comparator).
*/
private void writeObject(java.io.ObjectOutputStream s) throws java.io.IOException {
// Write out any hidden stuff
s.defaultWriteObject();
// Write out Comparator
s.writeObject(m.comparator());
// Write out size
s.writeInt(m.size());
// Write out all elements in the proper order.
for (Iterator i = m.keySet().iterator(); i.hasNext(); ) s.writeObject(i.next());
}
/**
* {@inheritDoc}
*
* @see java.util.NavigableSet#subSet(Object, boolean, Object, boolean)
* @since 1.6
*/
@SuppressWarnings("unchecked")
public NavigableSet<E> subSet(E start, boolean startInclusive, E end, boolean endInclusive) {
Comparator<? super E> c = backingMap.comparator();
int compare =
(c == null)
? ((net.sourceforge.retroweaver.harmony.runtime.java.lang.Comparable<E>) start)
.compareTo(end)
: c.compare(start, end);
if (compare <= 0) {
return new TreeSet<E>(backingMap.subMap(start, startInclusive, end, endInclusive));
}
throw new IllegalArgumentException();
}
@Override
public void convertLinks2Records() {
TreeMap<OIdentifiable, Change> newChanges = new TreeMap<OIdentifiable, Change>();
for (Map.Entry<OIdentifiable, Change> entry : changes.entrySet()) {
final OIdentifiable key = entry.getKey().getRecord();
if (key != null && this.owner != null) {
ORecordInternal.unTrack(this.owner, entry.getKey());
ORecordInternal.track(this.owner, key);
}
newChanges.put((key == null) ? entry.getKey() : key, entry.getValue());
}
changes.clear();
changes.putAll(newChanges);
}
@Override
public int deserialize(byte[] stream, int offset) {
final long fileId = OLongSerializer.INSTANCE.deserializeLiteral(stream, offset);
offset += OLongSerializer.LONG_SIZE;
final long pageIndex = OLongSerializer.INSTANCE.deserializeLiteral(stream, offset);
offset += OLongSerializer.LONG_SIZE;
final int pageOffset = OIntegerSerializer.INSTANCE.deserializeLiteral(stream, offset);
offset += OIntegerSerializer.INT_SIZE;
// Cached bag size. Not used after 1.7.5
offset += OIntegerSerializer.INT_SIZE;
if (fileId == -1) collectionPointer = null;
else
collectionPointer =
new OBonsaiCollectionPointer(fileId, new OBonsaiBucketPointer(pageIndex, pageOffset));
this.size = -1;
changes.putAll(ChangeSerializationHelper.INSTANCE.deserializeChanges(stream, offset));
return offset;
}
private Map.Entry<OIdentifiable, Integer> nextChangedNotRemovedSBTreeEntry(
Iterator<Map.Entry<OIdentifiable, Integer>> iterator) {
while (iterator.hasNext()) {
final Map.Entry<OIdentifiable, Integer> entry = iterator.next();
final Change change = changes.get(entry.getKey());
if (change == null) return entry;
final int newValue = change.applyTo(entry.getValue());
if (newValue > 0)
return new Map.Entry<OIdentifiable, Integer>() {
@Override
public OIdentifiable getKey() {
return entry.getKey();
}
@Override
public Integer getValue() {
return newValue;
}
@Override
public Integer setValue(Integer value) {
throw new UnsupportedOperationException();
}
};
}
return null;
}
/**
* @throws ClassCastException {@inheritDoc}
* @throws NullPointerException if {@code fromElement} or {@code toElement} is null and this set
* uses natural ordering, or its comparator does not permit null elements
* @throws IllegalArgumentException {@inheritDoc}
* @since 1.6
*/
public NavigableSet<E> subSet(
E fromElement, boolean fromInclusive, E toElement, boolean toInclusive) {
return new TreeSet<E>(
m.subMap(
fromElement, fromInclusive,
toElement, toInclusive));
}
public void confirmDelete() {
collectionPointer = null;
changes.clear();
newEntries.clear();
size = 0;
if (changeListeners != null) changeListeners.clear();
changeListeners = null;
}
@Override
public void remove() {
if (currentRemoved)
throw new IllegalStateException("Current element has already been removed");
if (currentValue == null)
throw new IllegalStateException("Next method was not called for given iterator");
if (removeFromNewEntries(currentValue)) {
if (size >= 0) size--;
} else {
Change counter = changedValues.get(currentValue);
if (counter != null) {
counter.decrement();
if (size >= 0)
if (counter.isUndefined()) size = -1;
else size--;
} else {
if (nextChange != null) {
changedValues.put(currentValue, new DiffChange(-1));
changedValuesIterator =
changedValues.tailMap(nextChange.getKey(), false).entrySet().iterator();
} else {
changedValues.put(currentValue, new DiffChange(-1));
}
size = -1;
}
}
if (OSBTreeRidBag.this.owner != null)
ORecordInternal.unTrack(OSBTreeRidBag.this.owner, currentValue);
if (updateOwner)
fireCollectionChangedEvent(
new OMultiValueChangeEvent<OIdentifiable, OIdentifiable>(
OMultiValueChangeEvent.OChangeType.REMOVE,
currentValue,
null,
currentValue,
false));
currentRemoved = true;
}
@Override
public <T> T getProperty(String key, Class<T> targetType, T defaultValue) {
Object originalValue = configMap.get(key);
if (originalValue == null && key.contains(".")) {
originalValue = configMap.navigate(key.split("\\."));
if (originalValue != null) {
try {
configMap.put(key, originalValue);
} catch (Exception e) {
// ignore
}
}
}
if (originalValue != null) {
T value = conversionService.convert(originalValue, targetType);
return DefaultGroovyMethods.asBoolean(value) ? value : defaultValue;
}
return defaultValue;
}
/**
* This method is going to take the optionSetIdNumbers objects with the page number is equals or
* higher that the endTemplateStartPage parameter
*
* @param inputDto
* @param endTemplateStartPage
* @return
*/
private static NavigableMap<String, OptionSetIdNumber> getGroupEndOptionSetNumbers(
LoadEndTemplatePatternInputDto inputDto, String endTemplateStartPage) {
NavigableMap<String, OptionSetIdNumber> groupEndOptionSetNumbers =
new TreeMap<String, OptionSetIdNumber>();
for (Map.Entry<String, OptionSetIdNumber> entry :
inputDto.getMapOptionSetIdNumber().entrySet()) {
int comparatorResult =
entry.getValue().getPageNumber().compareTo(Integer.valueOf(endTemplateStartPage));
if (comparatorResult >= 0) {
groupEndOptionSetNumbers.put(entry.getKey(), entry.getValue());
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"UtilityNavigationPointerEnd loadEndTemplatePattern() groupEndOptionSetNumbers : "
+ groupEndOptionSetNumbers);
}
return groupEndOptionSetNumbers;
}
@Override
public boolean contains(OIdentifiable identifiable) {
if (newEntries.containsKey(identifiable)) return true;
Change counter = changes.get(identifiable);
if (counter != null) {
AbsoluteChange absoluteValue = getAbsoluteValue(identifiable);
if (counter.isUndefined()) {
changes.put(identifiable, absoluteValue);
}
counter = absoluteValue;
} else {
counter = getAbsoluteValue(identifiable);
}
return counter.applyTo(0) > 0;
}
@Override
public void reset() {
newEntryIterator = newEntries.entrySet().iterator();
this.changedValuesIterator = changedValues.entrySet().iterator();
if (sbTreeIterator != null) this.sbTreeIterator.reset();
nextChange = nextChangedNotRemovedEntry(changedValuesIterator);
if (sbTreeIterator != null)
nextSBTreeEntry = nextChangedNotRemovedSBTreeEntry(sbTreeIterator);
}
@Override
public MedlineField selectValue(int n) {
MedlineField.Builder builder = getNewBuilder();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < n; i++) {
sb.append(model.ceilingEntry(DATA_GENERATOR.nextLong(0, totalWeight)).getValue());
if (i < n) {
sb.append(" ");
}
}
builder.addProperty(fieldName, sb.toString());
return builder.build();
}
private void addRequestEndPointUsingPath(
ContextMeta context,
ServiceMeta service,
ServiceMethodMeta method,
RequestMeta requestMeta,
String path) {
RequestMetaData metaData = new RequestMetaData(path, context, requestMeta, method, service);
if (requestMeta.getCallType() == CallType.ADDRESS) {
metaDataMap.put(path, metaData);
} else {
treeMap.put(path, metaData);
}
}
/**
* Adds all of the elements in the specified collection to this set.
*
* @param c collection containing elements to be added to this set
* @return {@code true} if this set changed as a result of the call
* @throws ClassCastException if the elements provided cannot be compared with the elements
* currently in the set
* @throws NullPointerException if the specified collection is null or if any element is null and
* this set uses natural ordering, or its comparator does not permit null elements
*/
public boolean addAll(Collection<? extends E> c) {
// Use linear-time version if applicable
if (m.size() == 0 && c.size() > 0 && c instanceof SortedSet && m instanceof TreeMap) {
SortedSet<? extends E> set = (SortedSet<? extends E>) c;
TreeMap<E, Object> map = (TreeMap<E, Object>) m;
Comparator<? super E> cc = (Comparator<? super E>) set.comparator();
Comparator<? super E> mc = map.comparator();
if (cc == mc || (cc != null && cc.equals(mc))) {
map.addAllForTreeSet(set, PRESENT);
return true;
}
}
return super.addAll(c);
}
protected void waitForSplitting(String tableName, int regionCount)
throws IOException, InterruptedException {
int regionCountActual = 0;
for (int i = 0; i < MAX_WAIT_ITERATION; i++) {
try (HTable table = new HTable(conf, tableName)) {
regionCountActual = 0;
NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations();
for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
HServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue());
for (HServerLoad.RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName()))
regionCountActual++;
}
}
if (regionCountActual == regionCount) {
return;
}
}
Thread.sleep(WAIT_INTERVAL);
}
Assert.assertEquals(getMethodName() + " failed - ", regionCount, regionCountActual);
}
public void testBackwardIteration() throws IOException {
final int records = 10000;
long seed = System.currentTimeMillis();
MersenneTwisterFast mersenneTwisterFast = new MersenneTwisterFast(1381162033616L);
System.out.println("testBackwardIteration seed : " + seed);
NavigableMap<OClusterPosition, byte[]> positionRecordMap = new TreeMap<OClusterPosition, byte[]>();
ORecordVersion recordVersion = OVersionFactory.instance().createVersion();
recordVersion.increment();
recordVersion.increment();
for (int i = 0; i < records; i++) {
int recordSize = mersenneTwisterFast.nextInt(2 * OClusterPage.MAX_RECORD_SIZE) + 1;
byte[] record = new byte[recordSize];
mersenneTwisterFast.nextBytes(record);
final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(record, recordVersion, (byte) 2);
positionRecordMap.put(physicalPosition.clusterPosition, record);
}
Iterator<OClusterPosition> positionIterator = positionRecordMap.keySet().iterator();
while (positionIterator.hasNext()) {
OClusterPosition clusterPosition = positionIterator.next();
if (mersenneTwisterFast.nextBoolean()) {
Assert.assertTrue(paginatedCluster.deleteRecord(clusterPosition));
positionIterator.remove();
}
}
OPhysicalPosition physicalPosition = new OPhysicalPosition();
physicalPosition.clusterPosition = OClusterPositionFactory.INSTANCE.valueOf(Long.MAX_VALUE);
OPhysicalPosition[] positions = paginatedCluster.floorPositions(physicalPosition);
Assert.assertTrue(positions.length > 0);
positionIterator = positionRecordMap.descendingKeySet().iterator();
int counter = 0;
while (positionIterator.hasNext()) {
Assert.assertTrue(positions.length > 0);
OClusterPosition testedPosition = positionIterator.next();
Assert.assertEquals(positions[positions.length - 1].clusterPosition, testedPosition);
OPhysicalPosition positionToFind = positions[positions.length - 1];
positions = paginatedCluster.lowerPositions(positionToFind);
counter++;
}
Assert.assertEquals(paginatedCluster.getEntries(), counter);
Assert.assertEquals(paginatedCluster.getFirstPosition(), positionRecordMap.firstKey());
Assert.assertEquals(paginatedCluster.getLastPosition(), positionRecordMap.lastKey());
}
private RIDBagIterator(
IdentityHashMap<OIdentifiable, OModifiableInteger> newEntries,
NavigableMap<OIdentifiable, Change> changedValues,
SBTreeMapEntryIterator sbTreeIterator,
boolean convertToRecord) {
newEntryIterator = newEntries.entrySet().iterator();
this.changedValues = changedValues;
this.convertToRecord = convertToRecord;
this.changedValuesIterator = changedValues.entrySet().iterator();
this.sbTreeIterator = sbTreeIterator;
nextChange = nextChangedNotRemovedEntry(changedValuesIterator);
if (sbTreeIterator != null)
nextSBTreeEntry = nextChangedNotRemovedSBTreeEntry(sbTreeIterator);
}
private AbsoluteChange getAbsoluteValue(OIdentifiable identifiable) {
final OSBTreeBonsai<OIdentifiable, Integer> tree = loadTree();
try {
Integer oldValue;
if (tree == null) oldValue = 0;
else oldValue = tree.get(identifiable);
if (oldValue == null) oldValue = 0;
final Change change = changes.get(identifiable);
return new AbsoluteChange(change == null ? oldValue : change.applyTo(oldValue));
} finally {
releaseTree();
}
}
private RequestMetaData doGet(final String path) {
RequestMetaData requestMetaData = metaDataMap.get(path);
if (requestMetaData == null) {
final Map.Entry<String, RequestMetaData> entry = treeMap.lowerEntry(path);
if (entry == null) {
return null;
}
if (path.startsWith(entry.getKey())) {
return entry.getValue();
} else {
return null;
}
} else {
return requestMetaData;
}
}