@Override
public void process() throws IOException {
long startTime = System.currentTimeMillis();
try {
Status status = this.splitTaskExecutor.exec(wal, mode, reporter);
switch (status) {
case DONE:
endTask(
zkw,
new SplitLogTask.Done(this.serverName, this.mode),
SplitLogCounters.tot_wkr_task_done,
curTask,
curTaskZKVersion.intValue());
break;
case PREEMPTED:
SplitLogCounters.tot_wkr_preempt_task.incrementAndGet();
LOG.warn("task execution prempted " + wal);
break;
case ERR:
if (server != null && !server.isStopped()) {
endTask(
zkw,
new SplitLogTask.Err(this.serverName, this.mode),
SplitLogCounters.tot_wkr_task_err,
curTask,
curTaskZKVersion.intValue());
break;
}
// if the RS is exiting then there is probably a tons of stuff
// that can go wrong. Resign instead of signaling error.
// $FALL-THROUGH$
case RESIGNED:
if (server != null && server.isStopped()) {
LOG.info("task execution interrupted because worker is exiting " + curTask);
}
endTask(
zkw,
new SplitLogTask.Resigned(this.serverName, this.mode),
SplitLogCounters.tot_wkr_task_resigned,
curTask,
curTaskZKVersion.intValue());
break;
}
} finally {
LOG.info(
"worker "
+ serverName
+ " done with task "
+ curTask
+ " in "
+ (System.currentTimeMillis() - startTime)
+ "ms");
this.inProgressTasks.decrementAndGet();
}
}
@Test
public void testHugeFile() throws Exception {
setupPartition(PARTITION_ID_2, 2);
int numMsgs = 1000;
int msgsPerLine = 10;
RecoveryManager rm = (RecoveryManager) ManagementContainer.getInstance().getRecoveryManager();
rm.setMsgIdPerLine(msgsPerLine);
String archiveName = "activerecovery.multilinetest.archive";
createActiveRecoveryTestFile(archiveName, msgsPerLine, numMsgs, PARTITION_ID_2);
MutableInt chunkPartition = new MutableInt(-1);
MutableLong chunkSeekChar = new MutableLong(-1);
TimeQueryArchiveBuilder builder;
do {
resetPipeline();
builder =
new TimeQueryArchiveBuilder(
1,
1,
m_pipeline,
m_quota,
archiveName,
chunkPartition.intValue(),
chunkSeekChar.longValue());
builder.processTimeQuery(1L, 2L);
parseChunkHint(builder.getChunkHint(), chunkPartition, chunkSeekChar);
builder.finish();
} while (builder.hasMoreChunks());
assertEquals(numMsgs, m_processedIds.size());
}
/**
* get the available agent count map in all regions of the user, including the free agents and
* user specified agents.
*
* @param user current user
* @return user available agent count map
*/
@Override
@Transactional
public Map<String, MutableInt> getUserAvailableAgentCountMap(User user) {
Set<String> regions = getRegions();
Map<String, MutableInt> availShareAgents = newHashMap(regions);
Map<String, MutableInt> availUserOwnAgent = newHashMap(regions);
for (String region : regions) {
availShareAgents.put(region, new MutableInt(0));
availUserOwnAgent.put(region, new MutableInt(0));
}
String myAgentSuffix = "_owned_" + user.getUserId();
for (AgentInfo agentInfo : getAllActiveAgentInfoFromDB()) {
// Skip the all agents which doesn't approved, is inactive or
// doesn't have region
// prefix.
if (!agentInfo.isApproved()) {
continue;
}
String fullRegion = agentInfo.getRegion();
String region = extractRegionFromAgentRegion(fullRegion);
if (StringUtils.isBlank(region) || !regions.contains(region)) {
continue;
}
// It's my own agent
if (fullRegion.endsWith(myAgentSuffix)) {
incrementAgentCount(availUserOwnAgent, region, user.getUserId());
} else if (fullRegion.contains("_owned_")) {
// If it's the others agent.. skip..
continue;
} else {
incrementAgentCount(availShareAgents, region, user.getUserId());
}
}
int maxAgentSizePerConsole = getMaxAgentSizePerConsole();
for (String region : regions) {
MutableInt mutableInt = availShareAgents.get(region);
int shareAgentCount = mutableInt.intValue();
mutableInt.setValue(Math.min(shareAgentCount, maxAgentSizePerConsole));
mutableInt.add(availUserOwnAgent.get(region));
}
return availShareAgents;
}
@Test
public void testLineLengthChangeSmallerFile() throws Exception {
setupPartition(PARTITION_ID_2, 2);
int numMsgs = 1000;
int msgsPerLineStart = 100;
int msgsPerLineEnd = 10;
RecoveryManager rm = (RecoveryManager) ManagementContainer.getInstance().getRecoveryManager();
rm.setMsgIdPerLine(msgsPerLineStart);
String archiveName = "activerecovery.multilinetest.archive";
createActiveRecoveryTestFile(archiveName, msgsPerLineStart, numMsgs, PARTITION_ID_2);
// set to smaller line length after file was created
rm.setMsgIdPerLine(msgsPerLineEnd);
MutableInt chunkPartition = new MutableInt(-1);
MutableLong chunkSeekChar = new MutableLong(-1);
TimeQueryArchiveBuilder builder;
int iterations = 0;
do {
resetPipeline();
builder =
new TimeQueryArchiveBuilder(
1,
1,
m_pipeline,
m_quota,
archiveName,
chunkPartition.intValue(),
chunkSeekChar.longValue());
builder.processTimeQuery(1L, 2L);
parseChunkHint(builder.getChunkHint(), chunkPartition, chunkSeekChar);
builder.finish();
iterations++;
} while (builder.hasMoreChunks());
assertEquals(
numMsgs / msgsPerLineStart,
iterations); // we'll ask for 10, but we'll get 100 in each batch
assertEquals(numMsgs, m_processedIds.size());
}
@Test
public void testWriteAheadLog() throws Exception {
final MutableInt flushCount = new MutableInt();
final MutableBoolean isClosed = new MutableBoolean(false);
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new FSStorageAgent(testMeta.dir, null));
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan plan = scm.getPhysicalPlan();
Journal j = scm.getJournal();
ByteArrayOutputStream bos =
new ByteArrayOutputStream() {
@Override
public void flush() throws IOException {
super.flush();
flushCount.increment();
}
@Override
public void close() throws IOException {
super.close();
isClosed.setValue(true);
}
};
j.setOutputStream(new DataOutputStream(bos));
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
String externalId = new MockContainer(scm, o1p1.getContainer()).container.getExternalId();
assertEquals("flush count", 1, flushCount.intValue());
o1p1.setState(PTOperator.State.ACTIVE);
assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
assertEquals("flush count", 2, flushCount.intValue());
assertEquals("is closed", false, isClosed.booleanValue());
// this will close the stream. There are 2 calls to flush() during the close() - one in Kryo
// Output and one
// in FilterOutputStream
j.setOutputStream(null);
assertEquals("flush count", 4, flushCount.intValue());
assertEquals("is closed", true, isClosed.booleanValue());
// output stream is closed, so state will be changed without recording it in the journal
o1p1.setState(PTOperator.State.INACTIVE);
assertEquals(PTOperator.State.INACTIVE, o1p1.getState());
assertEquals("flush count", 4, flushCount.intValue());
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
InetSocketAddress addr1 = InetSocketAddress.createUnresolved("host1", 1);
PTContainer c1 = plan.getContainers().get(0);
c1.setState(PTContainer.State.ALLOCATED);
c1.host = "host1";
c1.bufferServerAddress = addr1;
c1.setAllocatedMemoryMB(2);
c1.setRequiredMemoryMB(1);
c1.setAllocatedVCores(3);
c1.setRequiredVCores(4);
j.setOutputStream(new DataOutputStream(bos));
j.write(c1.getSetContainerState());
c1.setExternalId(null);
c1.setState(PTContainer.State.NEW);
c1.setExternalId(null);
c1.host = null;
c1.bufferServerAddress = null;
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(externalId, c1.getExternalId());
assertEquals(PTContainer.State.ALLOCATED, c1.getState());
assertEquals("host1", c1.host);
assertEquals(addr1, c1.bufferServerAddress);
assertEquals(1, c1.getRequiredMemoryMB());
assertEquals(2, c1.getAllocatedMemoryMB());
assertEquals(3, c1.getAllocatedVCores());
assertEquals(4, c1.getRequiredVCores());
j.write(scm.getSetOperatorProperty("o1", "maxTuples", "100"));
o1.setMaxTuples(10);
j.setOutputStream(null);
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(100, o1.getMaxTuples());
j.setOutputStream(new DataOutputStream(bos));
scm.setOperatorProperty("o1", "maxTuples", "10");
assertEquals(10, o1.getMaxTuples());
o1.setMaxTuples(100);
assertEquals(100, o1.getMaxTuples());
j.setOutputStream(null);
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(10, o1.getMaxTuples());
j.setOutputStream(new DataOutputStream(bos));
scm.setPhysicalOperatorProperty(o1p1.getId(), "maxTuples", "50");
}
/**
* Test.
*
* @throws IOException If an error occurs
*/
@Test
public void test() throws IOException {
final List<String> realList;
final List<String> simulatedList;
final DefaultCategoryDataset dataset;
final StringBuilder html;
final MutableInt actualPropertyCount;
final MutableInt remainingPropertyCount;
if (browserVersion_ == BrowserVersion.INTERNET_EXPLORER_6) {
realList = IE6_;
simulatedList = IE6_SIMULATED_;
dataset = CATEGORY_DATASET_IE6_;
html = IE6_HTML_;
actualPropertyCount = IE6_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = IE6_REMAINING_PROPERTY_COUNT_;
} else if (browserVersion_ == BrowserVersion.INTERNET_EXPLORER_7) {
realList = IE7_;
simulatedList = IE7_SIMULATED_;
dataset = CATEGORY_DATASET_IE7_;
html = IE7_HTML_;
actualPropertyCount = IE7_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = IE7_REMAINING_PROPERTY_COUNT_;
} else if (browserVersion_ == BrowserVersion.INTERNET_EXPLORER_8) {
realList = IE8_;
simulatedList = IE8_SIMULATED_;
dataset = CATEGORY_DATASET_IE8_;
html = IE8_HTML_;
actualPropertyCount = IE8_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = IE8_REMAINING_PROPERTY_COUNT_;
} else if (browserVersion_ == BrowserVersion.FIREFOX_2) {
realList = FF2_;
simulatedList = FF2_SIMULATED_;
dataset = CATEGORY_DATASET_FF2_;
html = FF2_HTML_;
actualPropertyCount = FF2_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = FF2_REMAINING_PROPERTY_COUNT_;
} else if (browserVersion_ == BrowserVersion.FIREFOX_3) {
realList = FF3_;
simulatedList = FF3_SIMULATED_;
dataset = CATEGORY_DATASET_FF3_;
html = FF3_HTML_;
actualPropertyCount = FF3_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = FF3_REMAINING_PROPERTY_COUNT_;
} else if (browserVersion_ == BrowserVersion.FIREFOX_3_6) {
realList = FF3_6_;
simulatedList = FF3_6_SIMULATED_;
dataset = CATEGORY_DATASET_FF3_6_;
html = FF3_6_HTML_;
actualPropertyCount = FF3_6_ACTUAL_PROPERTY_COUNT_;
remainingPropertyCount = FF3_6_REMAINING_PROPERTY_COUNT_;
} else {
fail("Unknown BrowserVersion " + browserVersion_);
return;
}
List<String> realProperties = Arrays.asList(getValueOf(realList, name_).split(","));
List<String> simulatedProperties = Arrays.asList(getValueOf(simulatedList, name_).split(","));
if (realProperties.size() == 1 && realProperties.get(0).length() == 0) {
realProperties = new ArrayList<String>();
}
if (simulatedProperties.size() == 1 && simulatedProperties.get(0).length() == 0) {
simulatedProperties = new ArrayList<String>();
}
final List<String> originalRealProperties = new ArrayList<String>(realProperties);
removeParentheses(realProperties);
removeParentheses(simulatedProperties);
final List<String> erroredProperties = new ArrayList<String>(simulatedProperties);
erroredProperties.removeAll(realProperties);
final List<String> implementedProperties = new ArrayList<String>(simulatedProperties);
implementedProperties.retainAll(realProperties);
dataset.addValue(implementedProperties.size(), "Implemented", name_);
dataset.addValue(
realProperties.size(),
browserVersion_.getNickname().replace("FF", "Firefox ").replace("IE", "Internet Explorer "),
name_);
dataset.addValue(erroredProperties.size(), "Should not be implemented", name_);
final List<String> remainingProperties = new ArrayList<String>(realProperties);
remainingProperties.removeAll(implementedProperties);
actualPropertyCount.add(realProperties.size());
remainingPropertyCount.add(remainingProperties.size());
if (LOG.isDebugEnabled()) {
LOG.debug(name_ + ':' + browserVersion_.getNickname() + ':' + realProperties);
LOG.debug("Remaining" + ':' + remainingProperties);
LOG.debug("Error" + ':' + erroredProperties);
}
appendHtml(html, originalRealProperties, simulatedProperties, erroredProperties);
if (dataset.getColumnCount() == IE7_.size()) {
saveChart(dataset);
html.append("<tr><td colspan='3' align='right'><b>Total Implemented: ")
.append(actualPropertyCount.intValue() - remainingPropertyCount.intValue())
.append(" / ")
.append(actualPropertyCount.intValue())
.append("</b></td></tr>");
html.append("</table>")
.append('\n')
.append("<br>")
.append("Legend:")
.append("<br>")
.append("<span style='color: blue'>")
.append("To be implemented")
.append("</span>")
.append("<br>")
.append("<span style='color: green'>")
.append("Implemented")
.append("</span>")
.append("<br>")
.append("<span style='color: red'>")
.append("Should not be implemented")
.append("</span>")
.append("</html>");
FileUtils.writeStringToFile(
new File(
getArtifactsDirectory() + "/properties-" + browserVersion_.getNickname() + ".html"),
html.toString());
}
}
@Test
public void testRecommender() throws Exception {
MutableInt recommendCount = new MutableInt();
Recommender mockRecommender = new MockRecommender(recommendCount);
Recommender cachingRecommender = new CachingRecommender(mockRecommender);
cachingRecommender.recommend(1, 1);
assertEquals(1, recommendCount.intValue());
cachingRecommender.recommend(2, 1);
assertEquals(2, recommendCount.intValue());
cachingRecommender.recommend(1, 1);
assertEquals(2, recommendCount.intValue());
cachingRecommender.recommend(2, 1);
assertEquals(2, recommendCount.intValue());
cachingRecommender.refresh(null);
cachingRecommender.recommend(1, 1);
assertEquals(3, recommendCount.intValue());
cachingRecommender.recommend(2, 1);
assertEquals(4, recommendCount.intValue());
cachingRecommender.recommend(3, 1);
assertEquals(5, recommendCount.intValue());
// Results from this recommend() method can be cached...
IDRescorer rescorer = NullRescorer.getItemInstance();
cachingRecommender.refresh(null);
cachingRecommender.recommend(1, 1, rescorer);
assertEquals(6, recommendCount.intValue());
cachingRecommender.recommend(2, 1, rescorer);
assertEquals(7, recommendCount.intValue());
cachingRecommender.recommend(1, 1, rescorer);
assertEquals(7, recommendCount.intValue());
cachingRecommender.recommend(2, 1, rescorer);
assertEquals(7, recommendCount.intValue());
// until you switch Rescorers
cachingRecommender.recommend(1, 1, null);
assertEquals(8, recommendCount.intValue());
cachingRecommender.recommend(2, 1, null);
assertEquals(9, recommendCount.intValue());
cachingRecommender.refresh(null);
cachingRecommender.estimatePreference(1, 1);
assertEquals(10, recommendCount.intValue());
cachingRecommender.estimatePreference(1, 2);
assertEquals(11, recommendCount.intValue());
cachingRecommender.estimatePreference(1, 2);
assertEquals(11, recommendCount.intValue());
}
/**
* Tests the use of a custom job manager.
*
* @throws Exception if an error occurs
*/
@Test
public void useCustomJobManager() throws Exception {
final MutableInt jobCount = new MutableInt(0);
final JavaScriptJobManager mgr =
new JavaScriptJobManager() {
/** Serial version UID. */
private static final long serialVersionUID = 4189494067589390155L;
/** {@inheritDoc} */
public int waitForJobsStartingBefore(final long delayMillis) {
return jobCount.intValue();
}
/** {@inheritDoc} */
public int waitForJobs(final long timeoutMillis) {
return jobCount.intValue();
}
/** {@inheritDoc} */
public void stopJob(final int id) {
// Empty.
}
/** {@inheritDoc} */
public void shutdown() {
// Empty.
}
/** {@inheritDoc} */
public void removeJob(final int id) {
// Empty.
}
/** {@inheritDoc} */
public void removeAllJobs() {
// Empty.
}
/** {@inheritDoc} */
public int getJobCount() {
return jobCount.intValue();
}
/** {@inheritDoc} */
public int addJob(final JavaScriptJob job, final Page page) {
jobCount.increment();
return jobCount.intValue();
}
};
final WebWindowListener listener =
new WebWindowListener() {
/** {@inheritDoc} */
public void webWindowOpened(final WebWindowEvent event) {
((WebWindowImpl) event.getWebWindow()).setJobManager(mgr);
}
/** {@inheritDoc} */
public void webWindowContentChanged(final WebWindowEvent event) {
// Empty.
}
/** {@inheritDoc} */
public void webWindowClosed(final WebWindowEvent event) {
// Empty.
}
};
final WebClient client = new WebClient();
client.addWebWindowListener(listener);
final TopLevelWindow window = (TopLevelWindow) client.getCurrentWindow();
window.setJobManager(mgr);
final MockWebConnection conn = new MockWebConnection();
conn.setDefaultResponse(
"<html><body><script>window.setTimeout('', 500);</script></body></html>");
client.setWebConnection(conn);
client.getPage(URL_FIRST);
assertEquals(1, jobCount.intValue());
client.getPage(URL_FIRST);
assertEquals(2, jobCount.intValue());
}
private EventProcessingResult doAggregateRawEventsInternal() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException(
"The cluster lock "
+ AGGREGATION_LOCK_NAME
+ " must be owned by the current thread and server");
}
if (!this.portalEventDimensionPopulator.isCheckedDimensions()) {
// First time aggregation has happened, run populateDimensions to ensure enough dimension data
// exists
final boolean populatedDimensions = this.portalEventAggregationManager.populateDimensions();
if (!populatedDimensions) {
this.logger.warn(
"Aborting raw event aggregation, populateDimensions returned false so the state of date/time dimensions is unknown");
return null;
}
}
// Flush any dimension creation before aggregation
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
final IEventAggregatorStatus eventAggregatorStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true);
// Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
final String previousServerName = eventAggregatorStatus.getServerName();
if (previousServerName != null && !serverName.equals(previousServerName)) {
this.logger.debug(
"Last aggregation run on {} clearing all aggregation caches", previousServerName);
final Session session = getEntityManager().unwrap(Session.class);
final Cache cache = session.getSessionFactory().getCache();
cache.evictEntityRegions();
}
eventAggregatorStatus.setServerName(serverName);
// Calculate date range for aggregation
DateTime lastAggregated = eventAggregatorStatus.getLastEventDate();
if (lastAggregated == null) {
lastAggregated = portalEventDao.getOldestPortalEventTimestamp();
// No portal events to aggregate, skip aggregation
if (lastAggregated == null) {
return new EventProcessingResult(0, null, null, true);
}
// First time aggregation has run, initialize the CLEAN_UNCLOSED status to save catch-up time
final IEventAggregatorStatus cleanUnclosedStatus =
eventAggregationManagementDao.getEventAggregatorStatus(
ProcessingType.CLEAN_UNCLOSED, true);
AggregationIntervalInfo oldestMinuteInterval =
this.intervalHelper.getIntervalInfo(AggregationInterval.MINUTE, lastAggregated);
cleanUnclosedStatus.setLastEventDate(oldestMinuteInterval.getStart().minusMinutes(1));
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
}
final DateTime newestEventTime =
DateTime.now().minus(this.aggregationDelay).secondOfMinute().roundFloorCopy();
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
final MutableInt events = new MutableInt();
final MutableObject lastEventDate = new MutableObject(newestEventTime);
boolean complete;
try {
currentThread.setName(currentName + "-" + lastAggregated + "_" + newestEventTime);
logger.debug(
"Starting aggregation of events between {} (inc) and {} (exc)",
lastAggregated,
newestEventTime);
// Do aggregation, capturing the start and end dates
eventAggregatorStatus.setLastStart(DateTime.now());
complete =
portalEventDao.aggregatePortalEvents(
lastAggregated,
newestEventTime,
this.eventAggregationBatchSize,
new AggregateEventsHandler(events, lastEventDate, eventAggregatorStatus));
eventAggregatorStatus.setLastEventDate((DateTime) lastEventDate.getValue());
eventAggregatorStatus.setLastEnd(DateTime.now());
} finally {
currentThread.setName(currentName);
}
// Store the results of the aggregation
eventAggregationManagementDao.updateEventAggregatorStatus(eventAggregatorStatus);
complete =
complete
&& (this.eventAggregationBatchSize <= 0
|| events.intValue() < this.eventAggregationBatchSize);
return new EventProcessingResult(
events.intValue(), lastAggregated, eventAggregatorStatus.getLastEventDate(), complete);
}
@Override
public Sequence<Result<SearchResultValue>> run(
final Query<Result<SearchResultValue>> input, Map<String, Object> responseContext) {
if (!(input instanceof SearchQuery)) {
throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class);
}
final SearchQuery query = (SearchQuery) input;
final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter());
final List<DimensionSpec> dimensions = query.getDimensions();
final SearchQuerySpec searchQuerySpec = query.getQuery();
final int limit = query.getLimit();
final boolean descending = query.isDescending();
// Closing this will cause segfaults in unit tests.
final QueryableIndex index = segment.asQueryableIndex();
if (index != null) {
final TreeMap<SearchHit, MutableInt> retVal =
Maps.newTreeMap(query.getSort().getComparator());
Iterable<DimensionSpec> dimsToSearch;
if (dimensions == null || dimensions.isEmpty()) {
dimsToSearch =
Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
} else {
dimsToSearch = dimensions;
}
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final ImmutableBitmap baseFilter =
filter == null
? null
: filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index));
for (DimensionSpec dimension : dimsToSearch) {
final Column column = index.getColumn(dimension.getDimension());
if (column == null) {
continue;
}
final BitmapIndex bitmapIndex = column.getBitmapIndex();
ExtractionFn extractionFn = dimension.getExtractionFn();
if (extractionFn == null) {
extractionFn = IdentityExtractionFn.getInstance();
}
if (bitmapIndex != null) {
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i)));
if (!searchQuerySpec.accept(dimVal)) {
continue;
}
ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (baseFilter != null) {
bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap));
}
if (bitmap.size() > 0) {
MutableInt counter = new MutableInt(bitmap.size());
MutableInt prev =
retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter);
if (prev != null) {
counter.add(prev.intValue());
}
if (retVal.size() >= limit) {
return makeReturnResult(limit, retVal);
}
}
}
}
}
return makeReturnResult(limit, retVal);
}
final StorageAdapter adapter = segment.asStorageAdapter();
if (adapter == null) {
log.makeAlert("WTF!? Unable to process search query on segment.")
.addData("segment", segment.getIdentifier())
.addData("query", query)
.emit();
throw new ISE(
"Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
}
final Iterable<DimensionSpec> dimsToSearch;
if (dimensions == null || dimensions.isEmpty()) {
dimsToSearch =
Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
} else {
dimsToSearch = dimensions;
}
final Sequence<Cursor> cursors =
adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL, descending);
final TreeMap<SearchHit, MutableInt> retVal =
cursors.accumulate(
Maps.<SearchHit, SearchHit, MutableInt>newTreeMap(query.getSort().getComparator()),
new Accumulator<TreeMap<SearchHit, MutableInt>, Cursor>() {
@Override
public TreeMap<SearchHit, MutableInt> accumulate(
TreeMap<SearchHit, MutableInt> set, Cursor cursor) {
if (set.size() >= limit) {
return set;
}
Map<String, DimensionSelector> dimSelectors = Maps.newHashMap();
for (DimensionSpec dim : dimsToSearch) {
dimSelectors.put(dim.getOutputName(), cursor.makeDimensionSelector(dim));
}
while (!cursor.isDone()) {
for (Map.Entry<String, DimensionSelector> entry : dimSelectors.entrySet()) {
final DimensionSelector selector = entry.getValue();
if (selector != null) {
final IndexedInts vals = selector.getRow();
for (int i = 0; i < vals.size(); ++i) {
final String dimVal = selector.lookupName(vals.get(i));
if (searchQuerySpec.accept(dimVal)) {
MutableInt counter = new MutableInt(1);
MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter);
if (prev != null) {
counter.add(prev.intValue());
}
if (set.size() >= limit) {
return set;
}
}
}
}
}
cursor.advance();
}
return set;
}
});
return makeReturnResult(limit, retVal);
}
