@Override
public DateTime getMaxTime() {
final IndexedLongs timestamps = index.getReadOnlyTimestamps();
final DateTime retVal = new DateTime(timestamps.get(timestamps.size() - 1));
Closeables.closeQuietly(timestamps);
return retVal;
}
@Override
public int getDimensionCardinality(String dimension) {
final Indexed<String> dimValueLookup = index.getDimValueLookup(dimension);
if (dimValueLookup == null) {
return 0;
}
return dimValueLookup.size();
}
@Override
public Interval getInterval() {
return index.getDataInterval();
}
@Override
public Offset getInvertedIndexOffset(String dimension, String value) {
return new ConciseOffset(index.getInvertedIndex(dimension, value));
}
@Override
public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value) {
return index.getInvertedIndex(dimension, value);
}
@Override
public Indexed<String> getDimensionValues(String dimension) {
return index.getDimValueLookup(dimension);
}
@Override
public int getNumRows() {
return index.getReadOnlyTimestamps().size();
}
/**
* This produces iterators of Cursor objects that must be fully processed (until isDone()
* returns true) before the next Cursor is processed. It is *not* safe to pass these cursors off
* to another thread for parallel processing
*
* @return
*/
@Override
public Iterator<Cursor> iterator() {
final Map<String, Object> metricCacheMap = Maps.newHashMap();
final IndexedLongs timestamps = index.getReadOnlyTimestamps();
final FunctionalIterator<Cursor> retVal =
FunctionalIterator.create(
gran.iterable(interval.getStartMillis(), interval.getEndMillis()).iterator())
.transform(
new Function<Long, Cursor>() {
private int currRow = 0;
@Override
public Cursor apply(final Long input) {
final long timeStart = Math.max(interval.getStartMillis(), input);
while (currRow < timestamps.size() && timestamps.get(currRow) < timeStart) {
++currRow;
}
return new Cursor() {
private final DateTime myBucket = gran.toDateTime(input);
private final long nextBucket =
Math.min(gran.next(myBucket.getMillis()), interval.getEndMillis());
private final int initRow = currRow;
@Override
public DateTime getTime() {
return myBucket;
}
@Override
public void advance() {
++currRow;
}
@Override
public boolean isDone() {
return currRow >= timestamps.size()
|| timestamps.get(currRow) >= nextBucket;
}
@Override
public void reset() {
currRow = initRow;
}
@Override
public DimensionSelector makeDimensionSelector(final String dimensionName) {
final Indexed<? extends IndexedInts> rowVals =
index.getDimColumn(dimensionName);
final Indexed<String> dimValueLookup =
index.getDimValueLookup(dimensionName);
if (rowVals == null) {
return null;
}
return new DimensionSelector() {
@Override
public IndexedInts getRow() {
return rowVals.get(currRow);
}
@Override
public int getValueCardinality() {
return dimValueLookup.size();
}
@Override
public String lookupName(int id) {
final String retVal = dimValueLookup.get(id);
return retVal == null ? "" : retVal;
}
@Override
public int lookupId(String name) {
return ("".equals(name))
? dimValueLookup.indexOf(null)
: dimValueLookup.indexOf(name);
}
};
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName) {
IndexedFloats cachedMetricVals =
(IndexedFloats) metricCacheMap.get(metricName);
if (cachedMetricVals == null) {
final MetricHolder metricHolder = index.getMetricHolder(metricName);
if (metricHolder != null) {
cachedMetricVals = metricHolder.getFloatType();
if (cachedMetricVals != null) {
metricCacheMap.put(metricName, cachedMetricVals);
}
}
}
if (cachedMetricVals == null) {
return new FloatMetricSelector() {
@Override
public float get() {
return 0.0f;
}
};
}
final IndexedFloats metricVals = cachedMetricVals;
return new FloatMetricSelector() {
@Override
public float get() {
return metricVals.get(currRow);
}
};
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName) {
Indexed cachedMetricVals = (Indexed) metricCacheMap.get(metricName);
if (cachedMetricVals == null) {
final MetricHolder metricHolder = index.getMetricHolder(metricName);
if (metricHolder != null) {
cachedMetricVals = metricHolder.getComplexType();
if (cachedMetricVals != null) {
metricCacheMap.put(metricName, cachedMetricVals);
}
}
}
if (cachedMetricVals == null) {
return null;
}
final Indexed metricVals = cachedMetricVals;
return new ComplexMetricSelector() {
@Override
public Class classOfObject() {
return metricVals.getClazz();
}
@Override
public Object get() {
return metricVals.get(currRow);
}
};
}
};
}
});
return MoreIterators.after(
retVal,
new Runnable() {
@Override
public void run() {
Closeables.closeQuietly(timestamps);
for (Object object : metricCacheMap.values()) {
if (object instanceof Closeable) {
Closeables.closeQuietly((Closeable) object);
}
}
}
});
}
@Override
public Iterator<Cursor> iterator() {
final Offset baseOffset = offset.clone();
final Map<String, Object> metricHolderCache = Maps.newHashMap();
final IndexedLongs timestamps = index.getReadOnlyTimestamps();
final FunctionalIterator<Cursor> retVal =
FunctionalIterator.create(
gran.iterable(interval.getStartMillis(), interval.getEndMillis()).iterator())
.transform(
new Function<Long, Cursor>() {
@Override
public Cursor apply(final Long input) {
final long timeStart = Math.max(interval.getStartMillis(), input);
while (baseOffset.withinBounds()
&& timestamps.get(baseOffset.getOffset()) < timeStart) {
baseOffset.increment();
}
final Offset offset =
new TimestampCheckingOffset(
baseOffset,
timestamps,
Math.min(interval.getEndMillis(), gran.next(timeStart)));
return new Cursor() {
private final Offset initOffset = offset.clone();
private final DateTime myBucket = gran.toDateTime(input);
private Offset cursorOffset = offset;
@Override
public DateTime getTime() {
return myBucket;
}
@Override
public void advance() {
cursorOffset.increment();
}
@Override
public boolean isDone() {
return !cursorOffset.withinBounds();
}
@Override
public void reset() {
cursorOffset = initOffset.clone();
}
@Override
public DimensionSelector makeDimensionSelector(final String dimensionName) {
final Indexed<? extends IndexedInts> rowVals =
index.getDimColumn(dimensionName);
final Indexed<String> dimValueLookup =
index.getDimValueLookup(dimensionName);
if (rowVals == null) {
return null;
}
return new DimensionSelector() {
@Override
public IndexedInts getRow() {
return rowVals.get(cursorOffset.getOffset());
}
@Override
public int getValueCardinality() {
return dimValueLookup.size();
}
@Override
public String lookupName(int id) {
final String retVal = dimValueLookup.get(id);
return retVal == null ? "" : retVal;
}
@Override
public int lookupId(String name) {
return ("".equals(name))
? dimValueLookup.indexOf(null)
: dimValueLookup.indexOf(name);
}
};
}
@Override
public FloatMetricSelector makeFloatMetricSelector(String metricName) {
IndexedFloats cachedMetricVals =
(IndexedFloats) metricHolderCache.get(metricName);
if (cachedMetricVals == null) {
MetricHolder holder = index.getMetricHolder(metricName);
if (holder != null) {
cachedMetricVals = holder.getFloatType();
metricHolderCache.put(metricName, cachedMetricVals);
}
}
if (cachedMetricVals == null) {
return new FloatMetricSelector() {
@Override
public float get() {
return 0.0f;
}
};
}
final IndexedFloats metricVals = cachedMetricVals;
return new FloatMetricSelector() {
@Override
public float get() {
return metricVals.get(cursorOffset.getOffset());
}
};
}
@Override
public ComplexMetricSelector makeComplexMetricSelector(String metricName) {
Indexed cachedMetricVals = (Indexed) metricHolderCache.get(metricName);
if (cachedMetricVals == null) {
MetricHolder holder = index.getMetricHolder(metricName);
if (holder != null) {
cachedMetricVals = holder.getComplexType();
metricHolderCache.put(metricName, cachedMetricVals);
}
}
if (cachedMetricVals == null) {
return null;
}
final Indexed metricVals = cachedMetricVals;
return new ComplexMetricSelector() {
@Override
public Class classOfObject() {
return metricVals.getClazz();
}
@Override
public Object get() {
return metricVals.get(cursorOffset.getOffset());
}
};
}
};
}
});
// This after call is not perfect, if there is an exception during processing, it will never
// get called,
// but it's better than nothing and doing this properly all the time requires a lot more
// fixerating
return MoreIterators.after(
retVal,
new Runnable() {
@Override
public void run() {
Closeables.closeQuietly(timestamps);
for (Object object : metricHolderCache.values()) {
if (object instanceof Closeable) {
Closeables.closeQuietly((Closeable) object);
}
}
}
});
}
@Override
public ImmutableConciseSet getInvertedIndex(String dimension, String dimVal) {
return index.getInvertedIndex(dimension, dimVal);
}
@Override
public Indexed<String> getAvailableDimensions() {
return index.getAvailableDimensions();
}
public static Index convertMMapToIndex(MMappedIndex mmappedIndex) {
Indexed<String> dimsIndexed = mmappedIndex.getAvailableDimensions();
String[] dimensions = new String[dimsIndexed.size()];
for (int i = 0; i < dimsIndexed.size(); ++i) {
dimensions[i] = dimsIndexed.get(i);
}
Indexed<String> metricsIndexed = mmappedIndex.getAvailableMetrics();
String[] metrics = new String[metricsIndexed.size()];
for (int i = 0; i < metricsIndexed.size(); ++i) {
metrics[i] = metricsIndexed.get(i);
}
IndexedLongs timeBuf = mmappedIndex.getReadOnlyTimestamps();
long[] timestamps = new long[timeBuf.size()];
timeBuf.fill(0, timestamps);
Closeables.closeQuietly(timeBuf);
Map<String, MetricHolder> metricVals = Maps.newLinkedHashMap();
for (String metric : metrics) {
MetricHolder holder = mmappedIndex.getMetricHolder(metric);
switch (holder.getType()) {
case FLOAT:
IndexedFloats mmappedFloats = holder.getFloatType();
float[] metricValsArray = new float[mmappedFloats.size()];
mmappedFloats.fill(0, metricValsArray);
Closeables.closeQuietly(mmappedFloats);
metricVals.put(
metric,
MetricHolder.floatMetric(
metric,
CompressedFloatsIndexedSupplier.fromFloatBuffer(
FloatBuffer.wrap(metricValsArray), ByteOrder.nativeOrder())));
break;
case COMPLEX:
Indexed complexObjects = holder.getComplexType();
Object[] vals = new Object[complexObjects.size()];
for (int i = 0; i < complexObjects.size(); ++i) {
vals[i] = complexObjects.get(i);
}
final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(holder.getTypeName());
if (serde == null) {
throw new ISE("Unknown type[%s]", holder.getTypeName());
}
metricVals.put(
metric,
MetricHolder.complexMetric(
metric,
holder.getTypeName(),
new ArrayIndexed(vals, serde.getObjectStrategy().getClazz())));
break;
}
}
Map<String, Map<String, Integer>> dimIdLookup = Maps.newHashMap();
Map<String, String[]> reverseDimLookup = Maps.newHashMap();
Map<String, ImmutableConciseSet[]> invertedIndexesMap = Maps.newHashMap();
Map<String, DimensionColumn> dimensionColumns = Maps.newHashMap();
for (String dimension : dimensions) {
final Indexed<String> dimValueLookup = mmappedIndex.getDimValueLookup(dimension);
String[] values = new String[dimValueLookup.size()];
for (int i = 0; i < dimValueLookup.size(); ++i) {
values[i] = dimValueLookup.get(i);
}
Map<String, Integer> lookupMap = Maps.newHashMapWithExpectedSize(dimValueLookup.size());
for (int i = 0; i < values.length; i++) {
lookupMap.put(values[i], i);
}
ImmutableConciseSet[] invertedIndexes = new ImmutableConciseSet[values.length];
final Indexed<String> dimValuesIndexed = mmappedIndex.getDimValueLookup(dimension);
for (int i = 0; i < dimValuesIndexed.size(); ++i) {
invertedIndexes[i] = mmappedIndex.getInvertedIndex(dimension, dimValuesIndexed.get(i));
}
int[] dimValues = new int[timestamps.length];
Map<List<Integer>, Integer> rowGroupings = Maps.newHashMap();
final Indexed<? extends IndexedInts> dimColumn = mmappedIndex.getDimColumn(dimension);
for (int i = 0; i < dimColumn.size(); ++i) {
int[] expansionValue = Indexedids.arrayFromIndexedInts(dimColumn.get(i));
Integer value = rowGroupings.get(Ints.asList(expansionValue));
if (value == null) {
value = rowGroupings.size();
rowGroupings.put(Ints.asList(expansionValue), value);
}
dimValues[i] = value;
}
int[][] expansionValues = new int[rowGroupings.size()][];
for (Map.Entry<List<Integer>, Integer> entry : rowGroupings.entrySet()) {
expansionValues[entry.getValue()] = Ints.toArray(entry.getKey());
}
dimIdLookup.put(dimension, lookupMap);
reverseDimLookup.put(dimension, values);
invertedIndexesMap.put(dimension, invertedIndexes);
dimensionColumns.put(dimension, new DimensionColumn(expansionValues, dimValues));
}
return new Index(
dimensions,
metrics,
mmappedIndex.getDataInterval(),
timestamps,
metricVals,
dimIdLookup,
reverseDimLookup,
invertedIndexesMap,
dimensionColumns);
}
