private ItemAndBroadcast getClosedEpisode(
Brand brand, ProgData progData, Channel channel, DateTimeZone zone, Timestamp updatedAt) {
String uri = CLOSED_EPISODE + getClosedPostfix(channel);
Maybe<Identified> resolvedContent =
contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();
Episode episode;
if (resolvedContent.hasValue() && resolvedContent.requireValue() instanceof Episode) {
episode = (Episode) resolvedContent.requireValue();
} else {
episode = (Episode) getBasicEpisode(progData, true);
}
episode.setCanonicalUri(uri);
episode.setCurie(CLOSED_CURIE + getClosedPostfix(channel));
episode.setTitle(progData.getTitle());
episode.setScheduleOnly(true);
episode.setMediaType(channel.getMediaType());
Version version = findBestVersion(episode.getVersions());
Broadcast broadcast = broadcast(progData, channel, zone, updatedAt);
addBroadcast(version, broadcast);
return new ItemAndBroadcast(episode, Maybe.just(broadcast));
}
private Optional<Brand> getBrandWithoutChannel(ProgData progData, Timestamp updatedAt) {
String brandId = progData.getSeriesId();
if (Strings.isNullOrEmpty(brandId) || Strings.isNullOrEmpty(brandId.trim())) {
return Optional.absent();
}
String brandUri = PaHelper.getBrandUri(brandId);
Alias brandAlias = PaHelper.getBrandAlias(brandId);
Maybe<Identified> possiblePrevious =
contentResolver.findByCanonicalUris(ImmutableList.of(brandUri)).getFirstValue();
Brand brand =
possiblePrevious.hasValue()
? (Brand) possiblePrevious.requireValue()
: new Brand(brandUri, "pa:b-" + brandId, Publisher.PA);
brand.addAlias(brandAlias);
brand.setTitle(progData.getTitle());
brand.setDescription(Strings.emptyToNull(progData.getSeriesSynopsis()));
setCertificate(progData, brand);
setGenres(progData, brand);
setTopicRefs(brand);
if (isClosedBrand(Optional.of(brand))) {
brand.setScheduleOnly(true);
}
brand.setLastUpdated(updatedAt.toDateTimeUTC());
return Optional.of(brand);
}
private ItemAndBroadcast getFilm(
ProgData progData, Channel channel, DateTimeZone zone, Timestamp updatedAt) {
String filmUri = PaHelper.getFilmUri(identifierFor(progData));
Maybe<Identified> possiblePreviousData =
contentResolver.findByCanonicalUris(ImmutableList.of(filmUri)).getFirstValue();
Film film;
if (possiblePreviousData.hasValue()) {
Identified previous = possiblePreviousData.requireValue();
if (previous instanceof Film) {
film = (Film) previous;
} else {
film = new Film();
Item.copyTo((Episode) previous, film);
}
} else {
film = getBasicFilm(progData);
}
film.addAlias(PaHelper.getFilmAlias(identifierFor(progData)));
Optional<String> rtFilmIdentifier = rtFilmIdentifierFor(progData);
if (rtFilmIdentifier.isPresent()) {
film.addAlias(PaHelper.getRtFilmAlias(rtFilmIdentifier.get()));
}
film.setAliasUrls(ImmutableSet.of(PaHelper.getAlias(progData.getProgId())));
Broadcast broadcast = setCommonDetails(progData, channel, zone, film, updatedAt);
if (progData.getFilmYear() != null
&& MoreStrings.containsOnlyAsciiDigits(progData.getFilmYear())) {
film.setYear(Integer.parseInt(progData.getFilmYear()));
}
return new ItemAndBroadcast(film, Maybe.just(broadcast));
}
private Item getBasicFilmWithoutBroadcast(ProgData progData) {
ImmutableList.Builder<String> uris = ImmutableList.builder();
Optional<String> rtFilmIdentifier = rtFilmIdentifierFor(progData);
// Previously when constructing the uri we would failover to ProgId if the rtFilmIdentifier
// was missing but still write to the same namespace. To avoid creating duplicates we have
// to replicate this old behavior and use the rtFilmIdentifier for resolution when its
// available
String legacyFilmUri =
rtFilmIdentifier.isPresent()
? PaHelper.getLegacyFilmUri(rtFilmIdentifier.get())
: PaHelper.getLegacyFilmUri(progData.getProgId());
uris.add(legacyFilmUri);
uris.add(PaHelper.getAlias(progData.getProgId()));
uris.add(PaHelper.getFilmUri(identifierFor(progData)));
Map<String, Identified> resolvedContent =
contentResolver.findByUris(uris.build()).asResolvedMap();
Film film = getFilmFromResolvedContent(progData, resolvedContent, legacyFilmUri);
if (progData.getFilmYear() != null
&& MoreStrings.containsOnlyAsciiDigits(progData.getFilmYear())) {
film.setYear(Integer.parseInt(progData.getFilmYear()));
}
return film;
}
private ScoredCandidates<Container> scoreContainers(
Multiset<String> parents, int children, ResultDescription desc) {
Builder<Container> candidates = DefaultScoredCandidates.fromSource(NAME);
ResolvedContent containers = resolver.findByCanonicalUris(parents.elementSet());
for (Multiset.Entry<String> parent : parents.entrySet()) {
Maybe<Identified> possibledContainer = containers.get(parent.getElement());
if (possibledContainer.hasValue()) {
Identified identified = possibledContainer.requireValue();
if (identified instanceof Container) {
Container container = (Container) identified;
Score score = score(parent.getCount(), children);
candidates.addEquivalent(container, score);
desc.appendText(
"%s: scored %s (%s)", container.getCanonicalUri(), score, container.getTitle());
} else {
desc.appendText("%s: %s not container", parent, identified.getClass().getSimpleName());
}
} else {
desc.appendText("%s: missing", parent);
}
}
return candidates.build();
}
private Item getBasicEpisodeWithoutBroadcast(ProgData progData, boolean isEpisode) {
String episodeUri = PaHelper.getEpisodeUri(identifierFor(progData));
Maybe<Identified> possiblePrevious =
contentResolver.findByCanonicalUris(ImmutableList.of(episodeUri)).getFirstValue();
Item item;
if (possiblePrevious.hasValue()) {
Item previous = (Item) possiblePrevious.requireValue();
if (!(previous instanceof Episode) && isEpisode) {
String message =
String.format(
"%s resolved as %s being ingested as Episode",
episodeUri, previous.getClass().getSimpleName());
adapterLog.record(warnEntry().withSource(getClass()).withDescription(message));
log.info(message);
item = convertItemToEpisode(previous);
} else if (previous instanceof Episode && !isEpisode) {
String message =
String.format(
"%s resolved as %s being ingested as Item",
episodeUri, previous.getClass().getSimpleName());
adapterLog.record(errorEntry().withSource(getClass()).withDescription(message));
log.info(message);
item = new Item();
Item.copyTo(previous, item);
} else {
item = previous;
}
} else {
item = getBasicEpisode(progData, isEpisode);
}
if (SCHEDULED_ONLY_EPISODE.equals(episodeUri)) {
item.setScheduleOnly(true);
}
item.addAlias(PaHelper.getEpisodeAlias(identifierFor(progData)));
try {
if (item instanceof Episode) {
Episode episode = (Episode) item;
episode.setSpecial(getBooleanValue(progData.getAttr().getSpecial()));
episode.setEpisodeNumber(episodeNumber(progData));
episode.setSeriesNumber(seriesNumber(progData));
}
} catch (NumberFormatException e) {
// sometimes we don't get valid numbers
log.warn("Failed to parse a numeric field for PA episode {}", episodeUri, e);
}
return item;
}
@Test
public void testScoresOneWhenSeriesContainerWithSeriesAndEpisodeCountsInRange() {
final Brand subject = brandWithSeries(5);
final Brand candidate = brandWithSeries(6);
when(contentResolver.findByCanonicalUris(
ImmutableList.copyOf(Iterables.transform(subject.getSeriesRefs(), SeriesRef.TO_URI))))
.thenReturn(ResolvedContent.builder().putAll(series(5)).build());
when(contentResolver.findByCanonicalUris(
ImmutableList.copyOf(Iterables.transform(candidate.getSeriesRefs(), SeriesRef.TO_URI))))
.thenReturn(ResolvedContent.builder().putAll(series(6)).build());
ScoredCandidates<Container> score =
scorer.score(subject, ImmutableSet.<Container>of(candidate), desc());
assertThat(Iterables.getOnlyElement(score.candidates().values()), is(Score.ONE));
}
private Optional<Series> getSeriesWithoutChannel(ProgData progData, Timestamp updatedAt) {
if (Strings.isNullOrEmpty(progData.getSeriesNumber())
|| Strings.isNullOrEmpty(progData.getSeriesId())) {
return Optional.absent();
}
String seriesUri = PaHelper.getSeriesUri(progData.getSeriesId(), progData.getSeriesNumber());
Alias seriesAlias = PaHelper.getSeriesAlias(progData.getSeriesId(), progData.getSeriesNumber());
Maybe<Identified> possiblePrevious =
contentResolver.findByCanonicalUris(ImmutableList.of(seriesUri)).getFirstValue();
Series series =
possiblePrevious.hasValue()
? (Series) possiblePrevious.requireValue()
: new Series(
seriesUri,
"pa:s-" + progData.getSeriesId() + "-" + progData.getSeriesNumber(),
Publisher.PA);
series.addAlias(seriesAlias);
if (progData.getEpisodeTotal() != null && progData.getEpisodeTotal().trim().length() > 0) {
try {
series.setTotalEpisodes(Integer.parseInt(progData.getEpisodeTotal().trim()));
} catch (NumberFormatException e) {
adapterLog.record(
warnEntry()
.withCause(e)
.withSource(getClass())
.withDescription(
"Couldn't parse episode_total %s", progData.getEpisodeTotal().trim()));
}
}
if (progData.getSeriesNumber() != null && progData.getSeriesNumber().trim().length() > 0) {
try {
series.withSeriesNumber(Integer.parseInt(progData.getSeriesNumber().trim()));
} catch (NumberFormatException e) {
adapterLog.record(
warnEntry()
.withCause(e)
.withSource(getClass())
.withDescription(
"Couldn't parse series_number %s", progData.getSeriesNumber().trim()));
}
}
series.setPublisher(Publisher.PA);
setCertificate(progData, series);
setGenres(progData, series);
setTopicRefs(series);
series.setLastUpdated(updatedAt.toDateTimeUTC());
return Optional.of(series);
}
private ItemAndBroadcast getEpisode(
ProgData progData,
Channel channel,
DateTimeZone zone,
boolean isEpisode,
Timestamp updatedAt) {
String episodeUri = PaHelper.getEpisodeUri(identifierFor(progData));
Maybe<Identified> possiblePrevious =
contentResolver.findByCanonicalUris(ImmutableList.of(episodeUri)).getFirstValue();
Item item;
if (possiblePrevious.hasValue()) {
item = (Item) possiblePrevious.requireValue();
if (!(item instanceof Episode) && isEpisode) {
log.record(
warnEntry()
.withSource(getClass())
.withDescription(
"%s resolved as %s being ingested as Episode",
episodeUri, item.getClass().getSimpleName()));
item = convertItemToEpisode(item);
} else if (item instanceof Episode && !isEpisode) {
log.record(
errorEntry()
.withSource(getClass())
.withDescription(
"%s resolved as %s being ingested as Item",
episodeUri, item.getClass().getSimpleName()));
}
} else {
item = getBasicEpisode(progData, isEpisode);
}
item.addAlias(PaHelper.getEpisodeAlias(identifierFor(progData)));
Broadcast broadcast = setCommonDetails(progData, channel, zone, item, updatedAt);
try {
if (item instanceof Episode) {
Episode episode = (Episode) item;
episode.setSpecial(getBooleanValue(progData.getAttr().getSpecial()));
episode.setEpisodeNumber(episodeNumber(progData));
episode.setSeriesNumber(seriesNumber(progData));
}
} catch (NumberFormatException e) {
// sometimes we don't get valid numbers
// log.
}
return new ItemAndBroadcast(item, Maybe.just(broadcast));
}
@Test
@SuppressWarnings("unchecked")
public void testScoresNullWhenCandidateHasNoSeries() {
Brand subject = brandWithSeries(1);
Brand candidate = brandWithSeries(0);
when(contentResolver.findByCanonicalUris(
ImmutableList.copyOf(Iterables.transform(subject.getSeriesRefs(), SeriesRef.TO_URI))))
.thenReturn(ResolvedContent.builder().putAll(series(1)).build());
ScoredCandidates<Container> score = scorer.score(subject, ImmutableSet.of(candidate), desc());
assertThat(Iterables.getOnlyElement(score.candidates().values()), is(Score.nullScore()));
verify(contentResolver).findByCanonicalUris((Iterable<String>) any());
}
private Optional<Identified> resolveContent(LookupEntry lookupEntry) {
Optional<Identified> identified =
Optional.ofNullable(
contentResolver
.findByCanonicalUris(Lists.newArrayList(lookupEntry.uri()))
.getFirstValue()
.valueOrNull());
// we throw lots of exceptions defensively
if (!identified.isPresent()) {
throw new NoSuchElementException(
String.format(
"Unable to resolve Item from ID %d, URI %s", lookupEntry.id(), lookupEntry.uri()));
}
return identified;
}
private Map<String, SeriesRef> getSeriesRefs(Container container) {
Map<String, SeriesRef> seriesRefs = Maps.newLinkedHashMap();
if (container instanceof Brand) {
ResolvedContent resolvedSeries =
resolver.findByCanonicalUris(
Iterables.transform(((Brand) container).getSeriesRefs(), SeriesRef.TO_URI));
for (Series series : Iterables.filter(resolvedSeries.getAllResolvedResults(), Series.class)) {
seriesRefs.put(series.getCanonicalUri(), series.seriesRef());
}
} else if (container instanceof Series) {
// if this is a top level series then all its children *series* ref is to this too.
seriesRefs.put(container.getCanonicalUri(), ((Series) container).seriesRef());
} else {
throw new IllegalArgumentException(
"Unexpected Container type: " + container.getClass().getSimpleName());
}
return seriesRefs;
}
private Brand getBrandSummary(ProgData progData, Brand brand, Timestamp updatedAt) {
String uri =
brand.getCanonicalUri().replace(Publisher.PA.key(), Publisher.PA_SERIES_SUMMARIES.key());
Maybe<Identified> maybeBrandSummary =
contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();
Brand brandSummary;
if (maybeBrandSummary.isNothing()) {
brandSummary = new Brand();
brandSummary.setCanonicalUri(uri);
brandSummary.setPublisher(Publisher.PA_SERIES_SUMMARIES);
brandSummary.setEquivalentTo(ImmutableSet.of(LookupRef.from(brand)));
} else {
brandSummary = (Brand) maybeBrandSummary.requireValue();
}
brandSummary.setLongDescription(progData.getSeriesSummary());
brandSummary.setLastUpdated(updatedAt.toDateTimeUTC());
return brandSummary;
}
private Multimap<String, ChildRef> getChildRefs(Container container) {
Multimap<String, ChildRef> seriesChildRefs = LinkedListMultimap.create();
ResolvedContent resolvedEpisodes =
resolver.findByCanonicalUris(
Iterables.transform(container.getChildRefs(), ChildRef.TO_URI));
for (Item item : Iterables.filter(resolvedEpisodes.getAllResolvedResults(), Item.class)) {
if (!item.isActivelyPublished()) {
continue;
}
ChildRef childRef = item.childRef();
if (item instanceof Episode && ((Episode) item).getSeriesRef() != null) {
String seriesUri = ((Episode) item).getSeriesRef().getUri();
seriesChildRefs.put(seriesUri, childRef);
} else {
seriesChildRefs.put("none", childRef);
}
}
return seriesChildRefs;
}
private Series getSeriesSummary(ProgData progData, Series series, Timestamp updatedAt) {
String uri =
series.getCanonicalUri().replace(Publisher.PA.key(), Publisher.PA_SERIES_SUMMARIES.key());
Maybe<Identified> maybeSeriesSummary =
contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();
Series seriesSummary;
if (maybeSeriesSummary.isNothing()) {
seriesSummary = new Series();
seriesSummary.setCanonicalUri(uri);
seriesSummary.setPublisher(Publisher.PA_SERIES_SUMMARIES);
seriesSummary.setEquivalentTo(ImmutableSet.of(LookupRef.from(series)));
} else {
seriesSummary = (Series) maybeSeriesSummary.requireValue();
}
seriesSummary.setLongDescription(progData.getSeason().getSeasonSummary());
seriesSummary.setLastUpdated(updatedAt.toDateTimeUTC());
return seriesSummary;
}
private Optional<Brand> getBrand(ProgData progData, Channel channel, Timestamp updatedAt) {
String brandId = progData.getSeriesId();
if (Strings.isNullOrEmpty(brandId) || Strings.isNullOrEmpty(brandId.trim())) {
return Optional.absent();
}
String brandUri = PaHelper.getBrandUri(brandId);
Alias brandAlias = PaHelper.getBrandAlias(brandId);
Maybe<Identified> possiblePrevious =
contentResolver.findByCanonicalUris(ImmutableList.of(brandUri)).getFirstValue();
Brand brand =
possiblePrevious.hasValue()
? (Brand) possiblePrevious.requireValue()
: new Brand(brandUri, "pa:b-" + brandId, Publisher.PA);
brand.addAlias(brandAlias);
brand.setTitle(progData.getTitle());
brand.setDescription(Strings.emptyToNull(progData.getSeriesSynopsis()));
brand.setSpecialization(specialization(progData, channel));
brand.setMediaType(channel.getMediaType());
setCertificate(progData, brand);
setGenres(progData, brand);
selectImages(
progData.getPictures(),
brand,
PA_PICTURE_TYPE_BRAND,
PA_PICTURE_TYPE_SERIES,
Maybe.<String>nothing());
if (isClosedBrand(Optional.of(brand))) {
brand.setScheduleOnly(true);
}
brand.setLastUpdated(updatedAt.toDateTimeUTC());
return Optional.of(brand);
}
private Iterable<Item> childrenOf(ImmutableList<String> candidates) {
List<Identified> resolvedContent =
contentResolver.findByCanonicalUris(candidates).getAllResolvedResults();
Iterable<Container> resolvedContainers = Iterables.filter(resolvedContent, Container.class);
return Iterables.concat(Iterables.transform(resolvedContainers, TO_ITEMS));
}
private Iterable<Episode> childrenOf(Container container) {
ImmutableList<ChildRef> childRefs = container.getChildRefs();
Iterable<String> childUris = Iterables.transform(childRefs, TO_URI);
ResolvedContent children = contentResolver.findByCanonicalUris(childUris);
return Iterables.filter(children.getAllResolvedResults(), Episode.class);
}
@Test
public void testRunTask() throws Exception {
AdapterLog log = mock(AdapterLog.class);
ContentResolver contentResolver = mock(ContentResolver.class);
ContentWriter contentWriter = mock(ContentWriter.class);
ContentGroupResolver groupResolver = mock(ContentGroupResolver.class);
ContentGroupWriter groupWriter = mock(ContentGroupWriter.class);
final AtomicInteger contentCounter = new AtomicInteger(0);
final AtomicInteger playlistCounter = new AtomicInteger(0);
when(contentResolver.findByCanonicalUris(anyCollection()))
.thenReturn(new ResolvedContent(Collections.EMPTY_MAP));
doAnswer(
new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
System.out.println(
ToStringBuilder.reflectionToString(
invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
contentCounter.incrementAndGet();
return null;
}
})
.when(contentWriter)
.createOrUpdate(any(Container.class));
doAnswer(
new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
System.out.println(
ToStringBuilder.reflectionToString(
invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
contentCounter.incrementAndGet();
return null;
}
})
.when(contentWriter)
.createOrUpdate(any(Item.class));
when(groupResolver.findByCanonicalUris(anyCollection()))
.thenReturn(new ResolvedContent(Collections.EMPTY_MAP));
doAnswer(
new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
System.out.println(
ToStringBuilder.reflectionToString(
invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
playlistCounter.incrementAndGet();
return null;
}
})
.when(groupWriter)
.createOrUpdate(any(ContentGroup.class));
TheSpaceUpdater updater =
new TheSpaceUpdater(
contentResolver,
contentWriter,
groupResolver,
groupWriter,
log,
null,
null,
"http://thespace.org");
updater.runTask();
System.out.println("Total contents: " + contentCounter.get());
System.out.println("Total playlists: " + playlistCounter.get());
}
private Maybe<Identified> resolveExisting(Content content) {
ImmutableSet<String> uris = ImmutableSet.of(content.getCanonicalUri());
ResolvedContent resolved = resolver.findByCanonicalUris(uris);
return resolved.get(content.getCanonicalUri());
}
@Override
public void handle(WsProgramme programme, Iterable<WsAudioItem> audioItems) {
checkNotNull(programme.getProgId());
checkNotNull(programme.getSeriesId());
String episodeUri = uriFor(programme);
Maybe<Identified> possibleEpisode =
resolver.findByCanonicalUris(ImmutableSet.of(episodeUri)).get(episodeUri);
Episode episode = null;
if (possibleEpisode.hasValue()) {
Identified resolved = possibleEpisode.requireValue();
if (resolved instanceof Episode) {
episode = (Episode) resolved;
} else {
log.record(
errorEntry()
.withDescription(
"Resolved %s for episode %s", resolved.getClass().getSimpleName(), episodeUri));
return;
}
} else {
episode = new Episode(episodeUri, curieFor(programme), WORLD_SERVICE);
}
episode.setParentRef(new ParentRef(uriForBrand(programme.getSeriesId())));
episode.setTitle(titleFrom(programme, audioItems));
episode.setDescription(programme.getSynopsis());
if (!Strings.isNullOrEmpty(programme.getEpisodeNo())
&& programme.getEpisodeNo().matches("\\d+")) {
episode.setEpisodeNumber(Integer.parseInt(programme.getEpisodeNo()));
}
episode.setGenres(WsGenre.genresForCode(programme.getGenreCode()));
episode.setMediaType(AUDIO);
episode.setSpecialization(RADIO);
if (!Iterables.isEmpty(audioItems)) {
for (WsAudioItem audioItem : audioItems) {
Version version = new Version();
if (!Strings.isNullOrEmpty(audioItem.getDuration())
&& audioItem.getDuration().matches("\\d+")) {
version.setDuration(new Duration(Long.parseLong(audioItem.getDuration())));
}
Policy policy = policyFor(audioItem);
String broadcastUri = audioItem.getLinkAudioBroadcastQuality();
if (!Strings.isNullOrEmpty(broadcastUri)) {
version.addManifestedAs(encodingFrom(policy, broadcastUri, MimeType.AUDIO_WAV));
}
String thumbnailUri = audioItem.getLinkAudioThumbnail();
if (!Strings.isNullOrEmpty(thumbnailUri)) {
version.addManifestedAs(encodingFrom(policy, thumbnailUri, MimeType.AUDIO_MP3));
}
if (!version.getManifestedAs().isEmpty()) {
episode.addVersion(version);
}
}
}
Broadcast broadcast = broadcastFrom(programme);
if (broadcast != null) {
Version version = Iterables.getFirst(episode.getVersions(), new Version());
if (version.getDuration() == null) {
version.setDuration(Duration.standardSeconds(broadcast.getBroadcastDuration()));
}
version.addBroadcast(broadcast);
}
writer.createOrUpdate(episode);
}
@Override
public Iterable<Item> apply(@Nullable Container input) {
Iterable<String> childUris = Iterables.transform(input.getChildRefs(), ChildRef.TO_URI);
ResolvedContent children = contentResolver.findByCanonicalUris(childUris);
return Iterables.filter(children.getAllResolvedResults(), Item.class);
}