@Override
public boolean convertRecords2Links() {
final Map<OIdentifiable, Change> newChangedValues = new HashMap<OIdentifiable, Change>();
for (Map.Entry<OIdentifiable, Change> entry : changes.entrySet()) {
OIdentifiable identifiable = entry.getKey();
if (identifiable instanceof ORecord) {
ORID identity = identifiable.getIdentity();
ORecord record = (ORecord) identifiable;
identity = record.getIdentity();
newChangedValues.put(identity, entry.getValue());
} else newChangedValues.put(entry.getKey().getIdentity(), entry.getValue());
}
for (Map.Entry<OIdentifiable, Change> entry : newChangedValues.entrySet()) {
if (entry.getKey() instanceof ORecord) {
ORecord record = (ORecord) entry.getKey();
newChangedValues.put(record, entry.getValue());
} else return false;
}
newEntries.clear();
changes.clear();
changes.putAll(newChangedValues);
return true;
}
@Override
public void reset() {
newEntryIterator = newEntries.entrySet().iterator();
this.changedValuesIterator = changedValues.entrySet().iterator();
if (sbTreeIterator != null) this.sbTreeIterator.reset();
nextChange = nextChangedNotRemovedEntry(changedValuesIterator);
if (sbTreeIterator != null)
nextSBTreeEntry = nextChangedNotRemovedSBTreeEntry(sbTreeIterator);
}
@Override
public void convertLinks2Records() {
TreeMap<OIdentifiable, Change> newChanges = new TreeMap<OIdentifiable, Change>();
for (Map.Entry<OIdentifiable, Change> entry : changes.entrySet()) {
final OIdentifiable key = entry.getKey().getRecord();
if (key != null && this.owner != null) {
ORecordInternal.unTrack(this.owner, entry.getKey());
ORecordInternal.track(this.owner, key);
}
newChanges.put((key == null) ? entry.getKey() : key, entry.getValue());
}
changes.clear();
changes.putAll(newChanges);
}
private RIDBagIterator(
IdentityHashMap<OIdentifiable, OModifiableInteger> newEntries,
NavigableMap<OIdentifiable, Change> changedValues,
SBTreeMapEntryIterator sbTreeIterator,
boolean convertToRecord) {
newEntryIterator = newEntries.entrySet().iterator();
this.changedValues = changedValues;
this.convertToRecord = convertToRecord;
this.changedValuesIterator = changedValues.entrySet().iterator();
this.sbTreeIterator = sbTreeIterator;
nextChange = nextChangedNotRemovedEntry(changedValuesIterator);
if (sbTreeIterator != null)
nextSBTreeEntry = nextChangedNotRemovedSBTreeEntry(sbTreeIterator);
}
protected void waitForSplitting(String tableName, int regionCount)
throws IOException, InterruptedException {
int regionCountActual = 0;
for (int i = 0; i < MAX_WAIT_ITERATION; i++) {
try (HTable table = new HTable(conf, tableName)) {
regionCountActual = 0;
NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations();
for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
HServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue());
for (HServerLoad.RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName()))
regionCountActual++;
}
}
if (regionCountActual == regionCount) {
return;
}
}
Thread.sleep(WAIT_INTERVAL);
}
Assert.assertEquals(getMethodName() + " failed - ", regionCount, regionCountActual);
}
@Override
public Set<Entry<String, Object>> entrySet() {
return configMap.entrySet();
}
/**
* This method is going to return a Map with the End template records loaded from the external
* BaseData xml file
*
* @param inputDto
* @return
*/
public static Map<String, NavigationPointerDto> loadEndTemplatePattern(
LoadEndTemplatePatternInputDto inputDto) {
Map<String, NavigationPointerDto> Navigation_BaseDataDto =
new LinkedHashMap<String, NavigationPointerDto>();
if (LOG.isDebugEnabled()) {
LOG.debug(
"UtilityNavigationPointerEnd loadEndTemplatePattern() inputDto.getMapOptionSetIdNumber() : "
+ inputDto.getMapOptionSetIdNumber());
}
UtilityNavigationPointerEnd.validateInputDto(inputDto);
if (UtilityNavigationPointerEnd.validateInputDtoData(inputDto)) {
try {
Map<String, BaseDataDto> listBaseData =
inputDto.getEndTemplateList().get(inputDto.getPrinterType());
/**
* We are going to group all the end BaseData Records on the map of navigateEndDataMap,
* using the page xml node data as a key and as a value the list of baseData records
*/
Map<String, Map<String, BaseDataDto>> navigateEndDataMap =
UtilityNavigationPointerEnd.groupLastRecords(listBaseData);
String endTemplateStartPage =
UtilityNavigationPointerEnd.getFirstEndTemplatePageNumber(inputDto);
int countInitOptionSetId =
UtilityNavigationPointerEnd.getLastFirstOptionSetId(inputDto, endTemplateStartPage);
int lastEndProdAccOptionSetId =
inputDto.getMapOptionSetIdNumber().get(endTemplateStartPage).getOptionSetId();
/*
if (LOG.isDebugEnabled()) {
LOG.debug("UtilityNavigationPointerEnd loadEndTemplatePattern() endTemplateStartPage : " + endTemplateStartPage);
LOG.debug("UtilityNavigationPointerEnd loadEndTemplatePattern() countInitOptionSetId : " + countInitOptionSetId);
LOG.debug("UtilityNavigationPointerEnd loadEndTemplatePattern() navigateEndDataMap.size : " + navigateEndDataMap.size());
LOG.debug("UtilityNavigationPointerEnd loadEndTemplatePattern() lastEndProdAccOptionSetId : " + lastEndProdAccOptionSetId);
}
*/
NavigableMap<String, OptionSetIdNumber> groupEndOptionSetNumbers =
UtilityNavigationPointerEnd.getGroupEndOptionSetNumbers(inputDto, endTemplateStartPage);
int procOptionSetIdProcInit =
inputDto.getMapOptionSetIdNumber().get("proc").getOptionSetId();
while (countInitOptionSetId <= lastEndProdAccOptionSetId) {
for (Map.Entry<String, OptionSetIdNumber> entryOptionSetId :
groupEndOptionSetNumbers.entrySet()) {
Map<String, String> mapOptionSetIdToUpdate = new LinkedHashMap<String, String>();
boolean panelCreated = false;
if (LOG.isDebugEnabled()) {
LOG.debug(
"UtilityNavigationPointerEnd loadEndTemplatePattern() entryOptionSetId.getKey() : "
+ entryOptionSetId.getKey());
LOG.debug(
"UtilityNavigationPointerEnd loadEndTemplatePattern() navigateEndDataMap : "
+ navigateEndDataMap);
}
for (Map.Entry<String, BaseDataDto> baseDataDto :
navigateEndDataMap.get(entryOptionSetId.getKey()).entrySet()) {
String nextPageValue = "";
NavigationPointerDto navigationPointerDto = new NavigationPointerDto();
navigationPointerDto.setDatasource("" + Rp3Constants.CODES.END_TEMPLATE_DATA_SOURCE);
navigationPointerDto.setDataSourceRefId(baseDataDto.getKey());
navigationPointerDto.setPage(String.valueOf(entryOptionSetId.getKey()));
navigationPointerDto.setStreamId(inputDto.getStreamId());
if (!"proc".equalsIgnoreCase(entryOptionSetId.getKey())) {
Map.Entry<String, OptionSetIdNumber> nextOptionSetIdPanelEntry =
groupEndOptionSetNumbers.higherEntry(entryOptionSetId.getKey());
if (nextOptionSetIdPanelEntry != null
&& !"proc".equalsIgnoreCase(nextOptionSetIdPanelEntry.getKey())) {
if (nextOptionSetIdPanelEntry != null && panelCreated == false) {
nextOptionSetIdPanelEntry.getValue().incrementOptionSetIdByOne();
nextPageValue =
nextOptionSetIdPanelEntry.getKey()
+ "."
+ nextOptionSetIdPanelEntry.getValue().getOptionSetId();
panelCreated = true;
} else if (nextOptionSetIdPanelEntry == null) {
nextPageValue = baseDataDto.getValue().getNextPanel();
} else if (panelCreated) {
nextPageValue =
nextOptionSetIdPanelEntry.getKey()
+ "."
+ nextOptionSetIdPanelEntry.getValue().getOptionSetId();
}
} else if (nextOptionSetIdPanelEntry != null
&& "proc".equalsIgnoreCase(nextOptionSetIdPanelEntry.getKey())) {
// We need to increment by one each proc number
nextOptionSetIdPanelEntry.getValue().incrementOptionSetIdByOne();
nextPageValue =
nextOptionSetIdPanelEntry.getKey()
+ "."
+ nextOptionSetIdPanelEntry.getValue().getOptionSetId();
}
navigationPointerDto.setPageOptionSetID(String.valueOf(countInitOptionSetId));
} else {
String pageToCheck =
baseDataDto.getValue().getPage()
+ "."
+ baseDataDto.getValue().getPageOptionSetID();
if (!mapOptionSetIdToUpdate.containsKey(pageToCheck)) {
mapOptionSetIdToUpdate.put(
pageToCheck, String.valueOf(++procOptionSetIdProcInit));
}
navigationPointerDto.setPageOptionSetID(mapOptionSetIdToUpdate.get(pageToCheck));
nextPageValue = baseDataDto.getValue().getNextPanel();
}
navigationPointerDto.setNextPage(nextPageValue);
navigationPointerDto.setPrinterType(inputDto.getPrinterType());
navigationPointerDto.inputType = baseDataDto.getValue().getHtmlInputType();
navigationPointerDto.webFriendlyName = baseDataDto.getValue().getLabel();
Navigation_BaseDataDto.put(UtilityRefIdGenerator.buildRefId(), navigationPointerDto);
}
}
countInitOptionSetId++;
}
} catch (RuntimeException re) {
LOG.error("Error UtilityTemplateEndPattern getEndTemplateMap() " + re.getMessage(), re);
StatusMessage statusMessage =
new StatusMessage(
Rp3Constants.CODES_MSG.STATUS_MSG_LEVEL_METHOD.toString(),
MsgConstants.UNPE.STATUS_MSG_ERROR.toString(),
Arrays.asList(re.getMessage()),
Integer.valueOf(Rp3Constants.CODES_MSG.STATUS_MSG_LEVEL_EXCEPTION.toString()));
inputDto.getStatusMessagingList().add(statusMessage);
}
}
return Navigation_BaseDataDto;
}
@Test
public void testRandom() throws Exception {
Directory directory = newDirectory();
final Random r = random();
final IndexWriterConfig iwc =
LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r))
.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setRAMBufferSizeMB(
scaledRandomIntBetween(16, 64)); // we might index a lot - don't go crazy here
RandomIndexWriter indexWriter = new RandomIndexWriter(r, directory, iwc);
int numUniqueChildValues = scaledRandomIntBetween(100, 2000);
String[] childValues = new String[numUniqueChildValues];
for (int i = 0; i < numUniqueChildValues; i++) {
childValues[i] = Integer.toString(i);
}
IntOpenHashSet filteredOrDeletedDocs = new IntOpenHashSet();
int childDocId = 0;
int numParentDocs = scaledRandomIntBetween(1, numUniqueChildValues);
ObjectObjectOpenHashMap<String, NavigableMap<String, FloatArrayList>> childValueToParentIds =
new ObjectObjectOpenHashMap<>();
for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) {
boolean markParentAsDeleted = rarely();
boolean filterMe = rarely();
String parent = Integer.toString(parentDocId);
Document document = new Document();
document.add(
new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.YES));
document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO));
if (markParentAsDeleted) {
filteredOrDeletedDocs.add(parentDocId);
document.add(new StringField("delete", "me", Field.Store.NO));
}
if (filterMe) {
filteredOrDeletedDocs.add(parentDocId);
document.add(new StringField("filter", "me", Field.Store.NO));
}
indexWriter.addDocument(document);
int numChildDocs = scaledRandomIntBetween(0, 100);
for (int i = 0; i < numChildDocs; i++) {
boolean markChildAsDeleted = rarely();
String childValue = childValues[random().nextInt(childValues.length)];
document = new Document();
document.add(
new StringField(
UidFieldMapper.NAME,
Uid.createUid("child", Integer.toString(childDocId++)),
Field.Store.NO));
document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO));
document.add(
new StringField(
ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO));
document.add(new StringField("field1", childValue, Field.Store.NO));
if (markChildAsDeleted) {
document.add(new StringField("delete", "me", Field.Store.NO));
}
indexWriter.addDocument(document);
if (!markChildAsDeleted) {
NavigableMap<String, FloatArrayList> parentIdToChildScores;
if (childValueToParentIds.containsKey(childValue)) {
parentIdToChildScores = childValueToParentIds.lget();
} else {
childValueToParentIds.put(childValue, parentIdToChildScores = new TreeMap<>());
}
if (!markParentAsDeleted && !filterMe) {
FloatArrayList childScores = parentIdToChildScores.get(parent);
if (childScores == null) {
parentIdToChildScores.put(parent, childScores = new FloatArrayList());
}
childScores.add(1f);
}
}
}
}
// Delete docs that are marked to be deleted.
indexWriter.deleteDocuments(new Term("delete", "me"));
indexWriter.commit();
IndexReader indexReader = DirectoryReader.open(directory);
IndexSearcher searcher = new IndexSearcher(indexReader);
Engine.Searcher engineSearcher =
new Engine.Searcher(ChildrenQueryTests.class.getSimpleName(), searcher);
((TestSearchContext) SearchContext.current())
.setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher));
int max = numUniqueChildValues / 4;
for (int i = 0; i < max; i++) {
// Simulate a parent update
if (random().nextBoolean()) {
final int numberOfUpdatableParents = numParentDocs - filteredOrDeletedDocs.size();
int numberOfUpdates =
RandomInts.randomIntBetween(
random(), 0, Math.min(numberOfUpdatableParents, TEST_NIGHTLY ? 25 : 5));
for (int j = 0; j < numberOfUpdates; j++) {
int parentId;
do {
parentId = random().nextInt(numParentDocs);
} while (filteredOrDeletedDocs.contains(parentId));
String parentUid = Uid.createUid("parent", Integer.toString(parentId));
indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, parentUid));
Document document = new Document();
document.add(new StringField(UidFieldMapper.NAME, parentUid, Field.Store.YES));
document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO));
indexWriter.addDocument(document);
}
indexReader.close();
indexReader = DirectoryReader.open(indexWriter.w, true);
searcher = new IndexSearcher(indexReader);
engineSearcher =
new Engine.Searcher(ChildrenConstantScoreQueryTests.class.getSimpleName(), searcher);
((TestSearchContext) SearchContext.current())
.setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher));
}
String childValue = childValues[random().nextInt(numUniqueChildValues)];
int shortCircuitParentDocSet = random().nextInt(numParentDocs);
ScoreType scoreType = ScoreType.values()[random().nextInt(ScoreType.values().length)];
// leave min/max set to 0 half the time
int minChildren = random().nextInt(2) * scaledRandomIntBetween(0, 110);
int maxChildren = random().nextInt(2) * scaledRandomIntBetween(minChildren, 110);
QueryBuilder queryBuilder =
hasChildQuery("child", constantScoreQuery(termQuery("field1", childValue)))
.scoreType(scoreType.name().toLowerCase(Locale.ENGLISH))
.minChildren(minChildren)
.maxChildren(maxChildren)
.setShortCircuitCutoff(shortCircuitParentDocSet);
// Using a FQ, will invoke / test the Scorer#advance(..) and also let the Weight#scorer not
// get live docs as acceptedDocs
queryBuilder = filteredQuery(queryBuilder, notFilter(termFilter("filter", "me")));
Query query = parseQuery(queryBuilder);
BitSetCollector collector = new BitSetCollector(indexReader.maxDoc());
int numHits = 1 + random().nextInt(25);
TopScoreDocCollector actualTopDocsCollector = TopScoreDocCollector.create(numHits);
searcher.search(query, MultiCollector.wrap(collector, actualTopDocsCollector));
FixedBitSet actualResult = collector.getResult();
FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc());
TopScoreDocCollector expectedTopDocsCollector = TopScoreDocCollector.create(numHits);
if (childValueToParentIds.containsKey(childValue)) {
LeafReader slowLeafReader = SlowCompositeReaderWrapper.wrap(indexReader);
final FloatArrayList[] scores = new FloatArrayList[slowLeafReader.maxDoc()];
Terms terms = slowLeafReader.terms(UidFieldMapper.NAME);
if (terms != null) {
NavigableMap<String, FloatArrayList> parentIdToChildScores = childValueToParentIds.lget();
TermsEnum termsEnum = terms.iterator(null);
DocsEnum docsEnum = null;
for (Map.Entry<String, FloatArrayList> entry : parentIdToChildScores.entrySet()) {
int count = entry.getValue().elementsCount;
if (count >= minChildren && (maxChildren == 0 || count <= maxChildren)) {
TermsEnum.SeekStatus seekStatus =
termsEnum.seekCeil(Uid.createUidAsBytes("parent", entry.getKey()));
if (seekStatus == TermsEnum.SeekStatus.FOUND) {
docsEnum =
termsEnum.docs(slowLeafReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE);
expectedResult.set(docsEnum.nextDoc());
scores[docsEnum.docID()] = new FloatArrayList(entry.getValue());
} else if (seekStatus == TermsEnum.SeekStatus.END) {
break;
}
}
}
}
MockScorer mockScorer = new MockScorer(scoreType);
final LeafCollector leafCollector =
expectedTopDocsCollector.getLeafCollector(slowLeafReader.getContext());
leafCollector.setScorer(mockScorer);
for (int doc = expectedResult.nextSetBit(0);
doc < slowLeafReader.maxDoc();
doc =
doc + 1 >= expectedResult.length()
? DocIdSetIterator.NO_MORE_DOCS
: expectedResult.nextSetBit(doc + 1)) {
mockScorer.scores = scores[doc];
leafCollector.collect(doc);
}
}
assertBitSet(actualResult, expectedResult, searcher);
assertTopDocs(actualTopDocsCollector.topDocs(), expectedTopDocsCollector.topDocs());
}
indexWriter.close();
indexReader.close();
directory.close();
}
