public static Path makeSegmentOutputPath(
Path basePath,
FileSystem fileSystem,
String dataSource,
String version,
Interval interval,
int partitionNum) {
Path outputPath = new Path(prependFSIfNullScheme(fileSystem, basePath), "./" + dataSource);
if ("hdfs".equals(fileSystem.getScheme())) {
outputPath =
new Path(
outputPath,
String.format(
"./%s_%s",
interval.getStart().toString(ISODateTimeFormat.basicDateTime()),
interval.getEnd().toString(ISODateTimeFormat.basicDateTime())));
outputPath = new Path(outputPath, version.replace(":", "_"));
} else {
outputPath =
new Path(
outputPath,
String.format(
"./%s_%s", interval.getStart().toString(), interval.getEnd().toString()));
outputPath = new Path(outputPath, String.format("./%s", version));
}
outputPath = new Path(outputPath, Integer.toString(partitionNum));
return outputPath;
}
public int getDaysInInterval(Interval interval) {
LocalDate beginDate =
getBeginDate().isBefore(interval.getStart().toLocalDate())
? interval.getStart().toLocalDate()
: getBeginDate();
LocalDate endDate =
getEndDate() == null || getEndDate().isAfter(interval.getEnd().toLocalDate())
? interval.getEnd().toLocalDate()
: getEndDate();
return Days.daysBetween(beginDate, endDate).getDays();
}
@Override
public void serialize(
final Interval value, final JsonGenerator gen, final SerializerProvider provider)
throws IOException {
gen.writeStartObject();
gen.writeStringField("startdatetime", formatter.print(value.getStart()));
gen.writeStringField("starttimezone", value.getStart().getZone().toString());
gen.writeStringField("enddatetime", formatter.print(value.getEnd()));
gen.writeStringField("endtimezone", value.getEnd().getZone().toString());
gen.writeEndObject();
}
private void addField(String name, Object value) {
if (value == null) {
return;
}
if (value instanceof LocalDate) { // TODO: is this correct? Shouldnt we use LocalTime here?
DateTime time = ((LocalDate) value).toDateTimeAtStartOfDay();
addDateTimeInUTC(name, time);
} else if (value instanceof Interval) {
Interval interval = (Interval) value;
DateTime start = interval.getStart();
DateTime end = interval.getEnd();
addDateTimeInUTC(name + "_start", start);
addDateTimeInUTC(name + "_end", end);
} else if (value instanceof ValueObject) {
ValueObject object = (ValueObject) value;
inputDocument.addField(name, object.getIndexingValue());
} else if (value instanceof AbstractEntity) {
AbstractEntity object = (AbstractEntity) value;
inputDocument.addField(name, object.getId());
} else if (value instanceof Collection) {
Collection list = (Collection) value;
if (!list.isEmpty()) {
for (Object object : list) {
addField(name, object);
}
}
} else {
inputDocument.addField(name, value.toString());
}
}
@Override
public Iterable<Cursor> makeCursors(Filter filter, Interval interval, QueryGranularity gran) {
Interval actualInterval = interval;
if (!actualInterval.overlaps(index.dataInterval)) {
return ImmutableList.of();
}
if (actualInterval.getStart().isBefore(index.dataInterval.getStart())) {
actualInterval = actualInterval.withStart(index.dataInterval.getStart());
}
if (actualInterval.getEnd().isAfter(index.dataInterval.getEnd())) {
actualInterval = actualInterval.withEnd(index.dataInterval.getEnd());
}
final Iterable<Cursor> iterable;
if (filter == null) {
iterable = new NoFilterCursorIterable(index, actualInterval, gran);
} else {
Offset offset = new ConciseOffset(filter.goConcise(new MMappedInvertedIndexSelector(index)));
iterable = new CursorIterable(index, actualInterval, gran, offset);
}
return FunctionalIterable.create(iterable).keep(Functions.<Cursor>identity());
}
// TODO report status effectively
@Override
protected void runTask() {
try {
LocalDate today = LocalDate.now(DateTimeZone.UTC);
LocalDate start = today.minusDays(minusDays);
LocalDate finish = today.plusDays(plusDays);
List<Channel> youViewChannels = channelResolver.getAllChannels();
UpdateProgress progress = UpdateProgress.START;
while (!start.isAfter(finish)) {
LocalDate end = start.plusDays(1);
for (Channel channel : youViewChannels) {
Interval interval =
new Interval(start.toDateTimeAtStartOfDay(), end.toDateTimeAtStartOfDay());
Document xml =
fetcher.getSchedule(interval.getStart(), interval.getEnd(), getYouViewId(channel));
Element root = xml.getRootElement();
Elements entries = root.getChildElements(ENTRY_KEY, root.getNamespaceURI(ATOM_PREFIX));
progress = progress.reduce(processor.process(channel, entries, interval));
reportStatus(progress.toString());
}
start = end;
}
} catch (Exception e) {
log.error("Exception when processing YouView schedule", e);
Throwables.propagate(e);
}
}
public boolean overlaps(final Interval interval) {
Interval regimeInterval = getInterval();
return getBeginDate() != null
&& ((regimeInterval != null && regimeInterval.overlaps(interval))
|| (regimeInterval == null
&& !getBeginDate().isAfter(interval.getEnd().toLocalDate())));
}
private List<TimelineObjectHolder<VersionType, ObjectType>> lookup(
Interval interval, boolean incompleteOk) {
List<TimelineObjectHolder<VersionType, ObjectType>> retVal =
new ArrayList<TimelineObjectHolder<VersionType, ObjectType>>();
NavigableMap<Interval, TimelineEntry> timeline =
(incompleteOk) ? incompletePartitionsTimeline : completePartitionsTimeline;
for (Map.Entry<Interval, TimelineEntry> entry : timeline.entrySet()) {
Interval timelineInterval = entry.getKey();
TimelineEntry val = entry.getValue();
if (timelineInterval.overlaps(interval)) {
retVal.add(
new TimelineObjectHolder<VersionType, ObjectType>(
timelineInterval, val.getVersion(), val.getPartitionHolder()));
}
}
if (retVal.isEmpty()) {
return retVal;
}
TimelineObjectHolder<VersionType, ObjectType> firstEntry = retVal.get(0);
if (interval.overlaps(firstEntry.getInterval())
&& interval.getStart().isAfter(firstEntry.getInterval().getStart())) {
retVal.set(
0,
new TimelineObjectHolder<VersionType, ObjectType>(
new Interval(interval.getStart(), firstEntry.getInterval().getEnd()),
firstEntry.getVersion(),
firstEntry.getObject()));
}
TimelineObjectHolder<VersionType, ObjectType> lastEntry = retVal.get(retVal.size() - 1);
if (interval.overlaps(lastEntry.getInterval())
&& interval.getEnd().isBefore(lastEntry.getInterval().getEnd())) {
retVal.set(
retVal.size() - 1,
new TimelineObjectHolder<VersionType, ObjectType>(
new Interval(lastEntry.getInterval().getStart(), interval.getEnd()),
lastEntry.getVersion(),
lastEntry.getObject()));
}
return retVal;
}
public static CharSequence format(Interval interval) {
StringBuilder sb = new StringBuilder();
try {
HOUR_MINUTE_FORMAT.printTo(sb, interval.getStart());
sb.append(" - ");
HOUR_MINUTE_FORMAT.printTo(sb, interval.getEnd());
} catch (IOException e) {
// Will never happen for StringBuilder
}
return sb;
}
@Override
public AggregateCount getCounts(String name, Interval interval, DateTimeField resolution) {
DateTime end = interval.getEnd();
Chronology c = interval.getChronology();
DurationField resolutionDuration = resolution.getDurationField();
long[] counts;
if (resolutionDuration.getUnitMillis() == DateTimeConstants.MILLIS_PER_MINUTE) {
// Iterate through each hour in the interval and load the minutes for it
MutableDateTime dt = new MutableDateTime(interval.getStart());
dt.setRounding(c.hourOfDay());
Duration step = Duration.standardHours(1);
List<long[]> hours = new ArrayList<long[]>();
while (dt.isBefore(end)) {
hours.add(getMinCountsForHour(name, dt));
dt.add(step);
}
counts =
MetricUtils.concatArrays(
hours,
interval.getStart().getMinuteOfHour(),
interval.toPeriod().toStandardMinutes().getMinutes() + 1,
60);
} else if (resolutionDuration.getUnitMillis() == DateTimeConstants.MILLIS_PER_HOUR) {
DateTime cursor = new DateTime(c.dayOfMonth().roundFloor(interval.getStart().getMillis()));
List<long[]> days = new ArrayList<long[]>();
Duration step = Duration.standardHours(24);
while (cursor.isBefore(end)) {
days.add(getHourCountsForDay(name, cursor));
cursor = cursor.plus(step);
}
counts =
MetricUtils.concatArrays(
days,
interval.getStart().getHourOfDay(),
interval.toPeriod().toStandardHours().getHours() + 1,
24);
} else {
throw new IllegalArgumentException("Only minute or hour resolution is currently supported");
}
return new AggregateCount(name, interval, counts, resolution);
}
public ActionForward sessionPostback(
ActionMapping mapping,
ActionForm actionForm,
HttpServletRequest request,
HttpServletResponse response) {
CandidateSearchBean search = getRenderedObject("search");
RenderUtils.invalidateViewState();
Interval interval = search.getSession().getCandidacyInterval();
if (search.getCutStart() == null
|| !interval.contains(search.getCutStart().toDateTimeAtStartOfDay())) {
search.setCutStart(interval.getStart().toLocalDate());
}
if (search.getCutEnd() == null
|| !interval.contains(search.getCutEnd().toDateTimeAtStartOfDay())) {
search.setCutEnd(interval.getEnd().toLocalDate());
}
if (interval.contains(new LocalDate().minusDays(1).toDateMidnight())) {
search.setCutEnd(new LocalDate().minusDays(1));
}
request.setAttribute("search", search);
return mapping.findForward("candidates");
}
public static void createIndexDrdFile(
byte versionId,
File inDir,
GenericIndexed<String> availableDimensions,
GenericIndexed<String> availableMetrics,
Interval dataInterval)
throws IOException {
File indexFile = new File(inDir, "index.drd");
FileChannel channel = null;
try {
channel = new FileOutputStream(indexFile).getChannel();
channel.write(ByteBuffer.wrap(new byte[] {versionId}));
availableDimensions.writeToChannel(channel);
availableMetrics.writeToChannel(channel);
serializerUtils.writeString(
channel, String.format("%s/%s", dataInterval.getStart(), dataInterval.getEnd()));
} finally {
Closeables.closeQuietly(channel);
channel = null;
}
IndexIO.checkFileSize(indexFile);
}
protected LogInterval visibleIntervalFor(Rectangle clipBounds) {
Interval interval = uiTimeScale.viewToModel(clipBounds);
return new LogInterval(
new LogInstant(interval.getStart().getMillis() - 1, 0),
new LogInstant(interval.getEnd().getMillis() + 1, Integer.MAX_VALUE));
}
public void setInterval(Interval interval) {
if (getType().isAbsence()) {
this.end = interval.getEnd().toDate();
this.start = interval.getStart().toDate();
} else throw new IllegalStateException("Intervals only valid for absences");
}
@Override
public String write(Interval arg0) throws YamlException {
return dateTimeMapper.write(arg0.getStart()) + "|" + dateTimeMapper.write(arg0.getEnd());
}
/**
* @param timeline
* @param key
* @param entry
* @return boolean flag indicating whether or not we inserted or discarded something
*/
private boolean addAtKey(
NavigableMap<Interval, TimelineEntry> timeline, Interval key, TimelineEntry entry) {
boolean retVal = false;
Interval currKey = key;
Interval entryInterval = entry.getTrueInterval();
if (!currKey.overlaps(entryInterval)) {
return false;
}
while (entryInterval != null && currKey != null && currKey.overlaps(entryInterval)) {
Interval nextKey = timeline.higherKey(currKey);
int versionCompare =
versionComparator.compare(entry.getVersion(), timeline.get(currKey).getVersion());
if (versionCompare < 0) {
if (currKey.contains(entryInterval)) {
return true;
} else if (currKey.getStart().isBefore(entryInterval.getStart())) {
entryInterval = new Interval(currKey.getEnd(), entryInterval.getEnd());
} else {
addIntervalToTimeline(
new Interval(entryInterval.getStart(), currKey.getStart()), entry, timeline);
if (entryInterval.getEnd().isAfter(currKey.getEnd())) {
entryInterval = new Interval(currKey.getEnd(), entryInterval.getEnd());
} else {
entryInterval = null; // discard this entry
}
}
} else if (versionCompare > 0) {
TimelineEntry oldEntry = timeline.remove(currKey);
if (currKey.contains(entryInterval)) {
addIntervalToTimeline(
new Interval(currKey.getStart(), entryInterval.getStart()), oldEntry, timeline);
addIntervalToTimeline(
new Interval(entryInterval.getEnd(), currKey.getEnd()), oldEntry, timeline);
addIntervalToTimeline(entryInterval, entry, timeline);
return true;
} else if (currKey.getStart().isBefore(entryInterval.getStart())) {
addIntervalToTimeline(
new Interval(currKey.getStart(), entryInterval.getStart()), oldEntry, timeline);
} else if (entryInterval.getEnd().isBefore(currKey.getEnd())) {
addIntervalToTimeline(
new Interval(entryInterval.getEnd(), currKey.getEnd()), oldEntry, timeline);
}
} else {
if (timeline.get(currKey).equals(entry)) {
// This occurs when restoring segments
timeline.remove(currKey);
} else {
throw new UnsupportedOperationException(
String.format(
"Cannot add overlapping segments [%s and %s] with the same version [%s]",
currKey, entryInterval, entry.getVersion()));
}
}
currKey = nextKey;
retVal = true;
}
addIntervalToTimeline(entryInterval, entry, timeline);
return retVal;
}
