예제 #1
0
  private ItemAndBroadcast getClosedEpisode(
      Brand brand, ProgData progData, Channel channel, DateTimeZone zone, Timestamp updatedAt) {
    String uri = CLOSED_EPISODE + getClosedPostfix(channel);
    Maybe<Identified> resolvedContent =
        contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();

    Episode episode;
    if (resolvedContent.hasValue() && resolvedContent.requireValue() instanceof Episode) {
      episode = (Episode) resolvedContent.requireValue();
    } else {
      episode = (Episode) getBasicEpisode(progData, true);
    }
    episode.setCanonicalUri(uri);
    episode.setCurie(CLOSED_CURIE + getClosedPostfix(channel));
    episode.setTitle(progData.getTitle());
    episode.setScheduleOnly(true);
    episode.setMediaType(channel.getMediaType());

    Version version = findBestVersion(episode.getVersions());

    Broadcast broadcast = broadcast(progData, channel, zone, updatedAt);
    addBroadcast(version, broadcast);

    return new ItemAndBroadcast(episode, Maybe.just(broadcast));
  }
예제 #2
0
  private Optional<Brand> getBrandWithoutChannel(ProgData progData, Timestamp updatedAt) {

    String brandId = progData.getSeriesId();
    if (Strings.isNullOrEmpty(brandId) || Strings.isNullOrEmpty(brandId.trim())) {
      return Optional.absent();
    }

    String brandUri = PaHelper.getBrandUri(brandId);
    Alias brandAlias = PaHelper.getBrandAlias(brandId);

    Maybe<Identified> possiblePrevious =
        contentResolver.findByCanonicalUris(ImmutableList.of(brandUri)).getFirstValue();

    Brand brand =
        possiblePrevious.hasValue()
            ? (Brand) possiblePrevious.requireValue()
            : new Brand(brandUri, "pa:b-" + brandId, Publisher.PA);

    brand.addAlias(brandAlias);
    brand.setTitle(progData.getTitle());
    brand.setDescription(Strings.emptyToNull(progData.getSeriesSynopsis()));
    setCertificate(progData, brand);
    setGenres(progData, brand);
    setTopicRefs(brand);

    if (isClosedBrand(Optional.of(brand))) {
      brand.setScheduleOnly(true);
    }

    brand.setLastUpdated(updatedAt.toDateTimeUTC());

    return Optional.of(brand);
  }
예제 #3
0
  private ItemAndBroadcast getFilm(
      ProgData progData, Channel channel, DateTimeZone zone, Timestamp updatedAt) {
    String filmUri = PaHelper.getFilmUri(identifierFor(progData));
    Maybe<Identified> possiblePreviousData =
        contentResolver.findByCanonicalUris(ImmutableList.of(filmUri)).getFirstValue();

    Film film;
    if (possiblePreviousData.hasValue()) {
      Identified previous = possiblePreviousData.requireValue();
      if (previous instanceof Film) {
        film = (Film) previous;
      } else {
        film = new Film();
        Item.copyTo((Episode) previous, film);
      }
    } else {
      film = getBasicFilm(progData);
    }
    film.addAlias(PaHelper.getFilmAlias(identifierFor(progData)));
    Optional<String> rtFilmIdentifier = rtFilmIdentifierFor(progData);
    if (rtFilmIdentifier.isPresent()) {
      film.addAlias(PaHelper.getRtFilmAlias(rtFilmIdentifier.get()));
    }

    film.setAliasUrls(ImmutableSet.of(PaHelper.getAlias(progData.getProgId())));

    Broadcast broadcast = setCommonDetails(progData, channel, zone, film, updatedAt);

    if (progData.getFilmYear() != null
        && MoreStrings.containsOnlyAsciiDigits(progData.getFilmYear())) {
      film.setYear(Integer.parseInt(progData.getFilmYear()));
    }

    return new ItemAndBroadcast(film, Maybe.just(broadcast));
  }
  private ScoredCandidates<Container> scoreContainers(
      Multiset<String> parents, int children, ResultDescription desc) {
    Builder<Container> candidates = DefaultScoredCandidates.fromSource(NAME);

    ResolvedContent containers = resolver.findByCanonicalUris(parents.elementSet());

    for (Multiset.Entry<String> parent : parents.entrySet()) {
      Maybe<Identified> possibledContainer = containers.get(parent.getElement());
      if (possibledContainer.hasValue()) {
        Identified identified = possibledContainer.requireValue();
        if (identified instanceof Container) {
          Container container = (Container) identified;
          Score score = score(parent.getCount(), children);
          candidates.addEquivalent(container, score);
          desc.appendText(
              "%s: scored %s (%s)", container.getCanonicalUri(), score, container.getTitle());
        } else {
          desc.appendText("%s: %s not container", parent, identified.getClass().getSimpleName());
        }
      } else {
        desc.appendText("%s: missing", parent);
      }
    }

    return candidates.build();
  }
예제 #5
0
  private Item getBasicEpisodeWithoutBroadcast(ProgData progData, boolean isEpisode) {
    String episodeUri = PaHelper.getEpisodeUri(identifierFor(progData));

    Maybe<Identified> possiblePrevious =
        contentResolver.findByCanonicalUris(ImmutableList.of(episodeUri)).getFirstValue();

    Item item;
    if (possiblePrevious.hasValue()) {
      Item previous = (Item) possiblePrevious.requireValue();

      if (!(previous instanceof Episode) && isEpisode) {
        String message =
            String.format(
                "%s resolved as %s being ingested as Episode",
                episodeUri, previous.getClass().getSimpleName());

        adapterLog.record(warnEntry().withSource(getClass()).withDescription(message));
        log.info(message);

        item = convertItemToEpisode(previous);
      } else if (previous instanceof Episode && !isEpisode) {
        String message =
            String.format(
                "%s resolved as %s being ingested as Item",
                episodeUri, previous.getClass().getSimpleName());

        adapterLog.record(errorEntry().withSource(getClass()).withDescription(message));
        log.info(message);

        item = new Item();
        Item.copyTo(previous, item);
      } else {
        item = previous;
      }
    } else {
      item = getBasicEpisode(progData, isEpisode);
    }

    if (SCHEDULED_ONLY_EPISODE.equals(episodeUri)) {
      item.setScheduleOnly(true);
    }

    item.addAlias(PaHelper.getEpisodeAlias(identifierFor(progData)));

    try {
      if (item instanceof Episode) {
        Episode episode = (Episode) item;
        episode.setSpecial(getBooleanValue(progData.getAttr().getSpecial()));
        episode.setEpisodeNumber(episodeNumber(progData));
        episode.setSeriesNumber(seriesNumber(progData));
      }
    } catch (NumberFormatException e) {
      // sometimes we don't get valid numbers
      log.warn("Failed to parse a numeric field for PA episode {}", episodeUri, e);
    }
    return item;
  }
  @Test
  public void testScoresOneWhenSeriesContainerWithSeriesAndEpisodeCountsInRange() {

    final Brand subject = brandWithSeries(5);
    final Brand candidate = brandWithSeries(6);

    when(contentResolver.findByCanonicalUris(
            ImmutableList.copyOf(Iterables.transform(subject.getSeriesRefs(), SeriesRef.TO_URI))))
        .thenReturn(ResolvedContent.builder().putAll(series(5)).build());
    when(contentResolver.findByCanonicalUris(
            ImmutableList.copyOf(Iterables.transform(candidate.getSeriesRefs(), SeriesRef.TO_URI))))
        .thenReturn(ResolvedContent.builder().putAll(series(6)).build());

    ScoredCandidates<Container> score =
        scorer.score(subject, ImmutableSet.<Container>of(candidate), desc());

    assertThat(Iterables.getOnlyElement(score.candidates().values()), is(Score.ONE));
  }
예제 #7
0
  private Optional<Series> getSeriesWithoutChannel(ProgData progData, Timestamp updatedAt) {
    if (Strings.isNullOrEmpty(progData.getSeriesNumber())
        || Strings.isNullOrEmpty(progData.getSeriesId())) {
      return Optional.absent();
    }
    String seriesUri = PaHelper.getSeriesUri(progData.getSeriesId(), progData.getSeriesNumber());
    Alias seriesAlias = PaHelper.getSeriesAlias(progData.getSeriesId(), progData.getSeriesNumber());

    Maybe<Identified> possiblePrevious =
        contentResolver.findByCanonicalUris(ImmutableList.of(seriesUri)).getFirstValue();

    Series series =
        possiblePrevious.hasValue()
            ? (Series) possiblePrevious.requireValue()
            : new Series(
                seriesUri,
                "pa:s-" + progData.getSeriesId() + "-" + progData.getSeriesNumber(),
                Publisher.PA);

    series.addAlias(seriesAlias);

    if (progData.getEpisodeTotal() != null && progData.getEpisodeTotal().trim().length() > 0) {
      try {
        series.setTotalEpisodes(Integer.parseInt(progData.getEpisodeTotal().trim()));
      } catch (NumberFormatException e) {
        adapterLog.record(
            warnEntry()
                .withCause(e)
                .withSource(getClass())
                .withDescription(
                    "Couldn't parse episode_total %s", progData.getEpisodeTotal().trim()));
      }
    }

    if (progData.getSeriesNumber() != null && progData.getSeriesNumber().trim().length() > 0) {
      try {
        series.withSeriesNumber(Integer.parseInt(progData.getSeriesNumber().trim()));
      } catch (NumberFormatException e) {
        adapterLog.record(
            warnEntry()
                .withCause(e)
                .withSource(getClass())
                .withDescription(
                    "Couldn't parse series_number %s", progData.getSeriesNumber().trim()));
      }
    }

    series.setPublisher(Publisher.PA);
    setCertificate(progData, series);
    setGenres(progData, series);
    setTopicRefs(series);

    series.setLastUpdated(updatedAt.toDateTimeUTC());

    return Optional.of(series);
  }
예제 #8
0
  private ItemAndBroadcast getEpisode(
      ProgData progData,
      Channel channel,
      DateTimeZone zone,
      boolean isEpisode,
      Timestamp updatedAt) {

    String episodeUri = PaHelper.getEpisodeUri(identifierFor(progData));
    Maybe<Identified> possiblePrevious =
        contentResolver.findByCanonicalUris(ImmutableList.of(episodeUri)).getFirstValue();

    Item item;
    if (possiblePrevious.hasValue()) {
      item = (Item) possiblePrevious.requireValue();
      if (!(item instanceof Episode) && isEpisode) {
        log.record(
            warnEntry()
                .withSource(getClass())
                .withDescription(
                    "%s resolved as %s being ingested as Episode",
                    episodeUri, item.getClass().getSimpleName()));
        item = convertItemToEpisode(item);
      } else if (item instanceof Episode && !isEpisode) {
        log.record(
            errorEntry()
                .withSource(getClass())
                .withDescription(
                    "%s resolved as %s being ingested as Item",
                    episodeUri, item.getClass().getSimpleName()));
      }
    } else {
      item = getBasicEpisode(progData, isEpisode);
    }

    item.addAlias(PaHelper.getEpisodeAlias(identifierFor(progData)));

    Broadcast broadcast = setCommonDetails(progData, channel, zone, item, updatedAt);

    try {
      if (item instanceof Episode) {
        Episode episode = (Episode) item;
        episode.setSpecial(getBooleanValue(progData.getAttr().getSpecial()));
        episode.setEpisodeNumber(episodeNumber(progData));
        episode.setSeriesNumber(seriesNumber(progData));
      }
    } catch (NumberFormatException e) {
      // sometimes we don't get valid numbers
      // log.
    }

    return new ItemAndBroadcast(item, Maybe.just(broadcast));
  }
  @Test
  @SuppressWarnings("unchecked")
  public void testScoresNullWhenCandidateHasNoSeries() {

    Brand subject = brandWithSeries(1);
    Brand candidate = brandWithSeries(0);

    when(contentResolver.findByCanonicalUris(
            ImmutableList.copyOf(Iterables.transform(subject.getSeriesRefs(), SeriesRef.TO_URI))))
        .thenReturn(ResolvedContent.builder().putAll(series(1)).build());

    ScoredCandidates<Container> score = scorer.score(subject, ImmutableSet.of(candidate), desc());

    assertThat(Iterables.getOnlyElement(score.candidates().values()), is(Score.nullScore()));
    verify(contentResolver).findByCanonicalUris((Iterable<String>) any());
  }
예제 #10
0
  private Optional<Identified> resolveContent(LookupEntry lookupEntry) {
    Optional<Identified> identified =
        Optional.ofNullable(
            contentResolver
                .findByCanonicalUris(Lists.newArrayList(lookupEntry.uri()))
                .getFirstValue()
                .valueOrNull());

    // we throw lots of exceptions defensively
    if (!identified.isPresent()) {
      throw new NoSuchElementException(
          String.format(
              "Unable to resolve Item from ID %d, URI %s", lookupEntry.id(), lookupEntry.uri()));
    }
    return identified;
  }
예제 #11
0
  private Map<String, SeriesRef> getSeriesRefs(Container container) {
    Map<String, SeriesRef> seriesRefs = Maps.newLinkedHashMap();

    if (container instanceof Brand) {
      ResolvedContent resolvedSeries =
          resolver.findByCanonicalUris(
              Iterables.transform(((Brand) container).getSeriesRefs(), SeriesRef.TO_URI));
      for (Series series : Iterables.filter(resolvedSeries.getAllResolvedResults(), Series.class)) {
        seriesRefs.put(series.getCanonicalUri(), series.seriesRef());
      }
    } else if (container instanceof Series) {
      // if this is a top level series then all its children *series* ref is to this too.
      seriesRefs.put(container.getCanonicalUri(), ((Series) container).seriesRef());
    } else {
      throw new IllegalArgumentException(
          "Unexpected Container type: " + container.getClass().getSimpleName());
    }
    return seriesRefs;
  }
예제 #12
0
  private Brand getBrandSummary(ProgData progData, Brand brand, Timestamp updatedAt) {
    String uri =
        brand.getCanonicalUri().replace(Publisher.PA.key(), Publisher.PA_SERIES_SUMMARIES.key());
    Maybe<Identified> maybeBrandSummary =
        contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();
    Brand brandSummary;

    if (maybeBrandSummary.isNothing()) {
      brandSummary = new Brand();
      brandSummary.setCanonicalUri(uri);
      brandSummary.setPublisher(Publisher.PA_SERIES_SUMMARIES);
      brandSummary.setEquivalentTo(ImmutableSet.of(LookupRef.from(brand)));
    } else {
      brandSummary = (Brand) maybeBrandSummary.requireValue();
    }

    brandSummary.setLongDescription(progData.getSeriesSummary());
    brandSummary.setLastUpdated(updatedAt.toDateTimeUTC());

    return brandSummary;
  }
예제 #13
0
  private Multimap<String, ChildRef> getChildRefs(Container container) {
    Multimap<String, ChildRef> seriesChildRefs = LinkedListMultimap.create();

    ResolvedContent resolvedEpisodes =
        resolver.findByCanonicalUris(
            Iterables.transform(container.getChildRefs(), ChildRef.TO_URI));
    for (Item item : Iterables.filter(resolvedEpisodes.getAllResolvedResults(), Item.class)) {
      if (!item.isActivelyPublished()) {
        continue;
      }

      ChildRef childRef = item.childRef();
      if (item instanceof Episode && ((Episode) item).getSeriesRef() != null) {
        String seriesUri = ((Episode) item).getSeriesRef().getUri();
        seriesChildRefs.put(seriesUri, childRef);
      } else {
        seriesChildRefs.put("none", childRef);
      }
    }
    return seriesChildRefs;
  }
예제 #14
0
  private Series getSeriesSummary(ProgData progData, Series series, Timestamp updatedAt) {
    String uri =
        series.getCanonicalUri().replace(Publisher.PA.key(), Publisher.PA_SERIES_SUMMARIES.key());
    Maybe<Identified> maybeSeriesSummary =
        contentResolver.findByCanonicalUris(ImmutableList.of(uri)).getFirstValue();
    Series seriesSummary;

    if (maybeSeriesSummary.isNothing()) {
      seriesSummary = new Series();
      seriesSummary.setCanonicalUri(uri);
      seriesSummary.setPublisher(Publisher.PA_SERIES_SUMMARIES);
      seriesSummary.setEquivalentTo(ImmutableSet.of(LookupRef.from(series)));
    } else {
      seriesSummary = (Series) maybeSeriesSummary.requireValue();
    }

    seriesSummary.setLongDescription(progData.getSeason().getSeasonSummary());
    seriesSummary.setLastUpdated(updatedAt.toDateTimeUTC());

    return seriesSummary;
  }
예제 #15
0
  private Optional<Brand> getBrand(ProgData progData, Channel channel, Timestamp updatedAt) {
    String brandId = progData.getSeriesId();
    if (Strings.isNullOrEmpty(brandId) || Strings.isNullOrEmpty(brandId.trim())) {
      return Optional.absent();
    }

    String brandUri = PaHelper.getBrandUri(brandId);
    Alias brandAlias = PaHelper.getBrandAlias(brandId);

    Maybe<Identified> possiblePrevious =
        contentResolver.findByCanonicalUris(ImmutableList.of(brandUri)).getFirstValue();

    Brand brand =
        possiblePrevious.hasValue()
            ? (Brand) possiblePrevious.requireValue()
            : new Brand(brandUri, "pa:b-" + brandId, Publisher.PA);

    brand.addAlias(brandAlias);
    brand.setTitle(progData.getTitle());
    brand.setDescription(Strings.emptyToNull(progData.getSeriesSynopsis()));
    brand.setSpecialization(specialization(progData, channel));
    brand.setMediaType(channel.getMediaType());
    setCertificate(progData, brand);
    setGenres(progData, brand);

    selectImages(
        progData.getPictures(),
        brand,
        PA_PICTURE_TYPE_BRAND,
        PA_PICTURE_TYPE_SERIES,
        Maybe.<String>nothing());

    if (isClosedBrand(Optional.of(brand))) {
      brand.setScheduleOnly(true);
    }
    brand.setLastUpdated(updatedAt.toDateTimeUTC());

    return Optional.of(brand);
  }
 private Iterable<Item> childrenOf(ImmutableList<String> candidates) {
   List<Identified> resolvedContent =
       contentResolver.findByCanonicalUris(candidates).getAllResolvedResults();
   Iterable<Container> resolvedContainers = Iterables.filter(resolvedContent, Container.class);
   return Iterables.concat(Iterables.transform(resolvedContainers, TO_ITEMS));
 }
 private Iterable<Episode> childrenOf(Container container) {
   ImmutableList<ChildRef> childRefs = container.getChildRefs();
   Iterable<String> childUris = Iterables.transform(childRefs, TO_URI);
   ResolvedContent children = contentResolver.findByCanonicalUris(childUris);
   return Iterables.filter(children.getAllResolvedResults(), Episode.class);
 }
예제 #18
0
  @Test
  public void testRunTask() throws Exception {
    AdapterLog log = mock(AdapterLog.class);
    ContentResolver contentResolver = mock(ContentResolver.class);
    ContentWriter contentWriter = mock(ContentWriter.class);
    ContentGroupResolver groupResolver = mock(ContentGroupResolver.class);
    ContentGroupWriter groupWriter = mock(ContentGroupWriter.class);

    final AtomicInteger contentCounter = new AtomicInteger(0);
    final AtomicInteger playlistCounter = new AtomicInteger(0);

    when(contentResolver.findByCanonicalUris(anyCollection()))
        .thenReturn(new ResolvedContent(Collections.EMPTY_MAP));
    doAnswer(
            new Answer() {

              @Override
              public Object answer(InvocationOnMock invocation) throws Throwable {
                System.out.println(
                    ToStringBuilder.reflectionToString(
                        invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
                contentCounter.incrementAndGet();
                return null;
              }
            })
        .when(contentWriter)
        .createOrUpdate(any(Container.class));
    doAnswer(
            new Answer() {

              @Override
              public Object answer(InvocationOnMock invocation) throws Throwable {
                System.out.println(
                    ToStringBuilder.reflectionToString(
                        invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
                contentCounter.incrementAndGet();
                return null;
              }
            })
        .when(contentWriter)
        .createOrUpdate(any(Item.class));

    when(groupResolver.findByCanonicalUris(anyCollection()))
        .thenReturn(new ResolvedContent(Collections.EMPTY_MAP));
    doAnswer(
            new Answer() {

              @Override
              public Object answer(InvocationOnMock invocation) throws Throwable {
                System.out.println(
                    ToStringBuilder.reflectionToString(
                        invocation.getArguments()[0], ToStringStyle.MULTI_LINE_STYLE));
                playlistCounter.incrementAndGet();
                return null;
              }
            })
        .when(groupWriter)
        .createOrUpdate(any(ContentGroup.class));

    TheSpaceUpdater updater =
        new TheSpaceUpdater(
            contentResolver,
            contentWriter,
            groupResolver,
            groupWriter,
            log,
            null,
            null,
            "http://thespace.org");
    updater.runTask();

    System.out.println("Total contents: " + contentCounter.get());
    System.out.println("Total playlists: " + playlistCounter.get());
  }
예제 #19
0
 private Maybe<Identified> resolveExisting(Content content) {
   ImmutableSet<String> uris = ImmutableSet.of(content.getCanonicalUri());
   ResolvedContent resolved = resolver.findByCanonicalUris(uris);
   return resolved.get(content.getCanonicalUri());
 }
예제 #20
0
  @Override
  public void handle(WsProgramme programme, Iterable<WsAudioItem> audioItems) {
    checkNotNull(programme.getProgId());
    checkNotNull(programme.getSeriesId());

    String episodeUri = uriFor(programme);

    Maybe<Identified> possibleEpisode =
        resolver.findByCanonicalUris(ImmutableSet.of(episodeUri)).get(episodeUri);

    Episode episode = null;

    if (possibleEpisode.hasValue()) {
      Identified resolved = possibleEpisode.requireValue();
      if (resolved instanceof Episode) {
        episode = (Episode) resolved;
      } else {
        log.record(
            errorEntry()
                .withDescription(
                    "Resolved %s for episode %s", resolved.getClass().getSimpleName(), episodeUri));
        return;
      }
    } else {
      episode = new Episode(episodeUri, curieFor(programme), WORLD_SERVICE);
    }

    episode.setParentRef(new ParentRef(uriForBrand(programme.getSeriesId())));
    episode.setTitle(titleFrom(programme, audioItems));
    episode.setDescription(programme.getSynopsis());
    if (!Strings.isNullOrEmpty(programme.getEpisodeNo())
        && programme.getEpisodeNo().matches("\\d+")) {
      episode.setEpisodeNumber(Integer.parseInt(programme.getEpisodeNo()));
    }
    episode.setGenres(WsGenre.genresForCode(programme.getGenreCode()));
    episode.setMediaType(AUDIO);
    episode.setSpecialization(RADIO);

    if (!Iterables.isEmpty(audioItems)) {
      for (WsAudioItem audioItem : audioItems) {
        Version version = new Version();

        if (!Strings.isNullOrEmpty(audioItem.getDuration())
            && audioItem.getDuration().matches("\\d+")) {
          version.setDuration(new Duration(Long.parseLong(audioItem.getDuration())));
        }

        Policy policy = policyFor(audioItem);

        String broadcastUri = audioItem.getLinkAudioBroadcastQuality();
        if (!Strings.isNullOrEmpty(broadcastUri)) {
          version.addManifestedAs(encodingFrom(policy, broadcastUri, MimeType.AUDIO_WAV));
        }

        String thumbnailUri = audioItem.getLinkAudioThumbnail();
        if (!Strings.isNullOrEmpty(thumbnailUri)) {
          version.addManifestedAs(encodingFrom(policy, thumbnailUri, MimeType.AUDIO_MP3));
        }
        if (!version.getManifestedAs().isEmpty()) {
          episode.addVersion(version);
        }
      }
    }

    Broadcast broadcast = broadcastFrom(programme);
    if (broadcast != null) {
      Version version = Iterables.getFirst(episode.getVersions(), new Version());
      if (version.getDuration() == null) {
        version.setDuration(Duration.standardSeconds(broadcast.getBroadcastDuration()));
      }
      version.addBroadcast(broadcast);
    }

    writer.createOrUpdate(episode);
  }
 @Override
 public Iterable<Item> apply(@Nullable Container input) {
   Iterable<String> childUris = Iterables.transform(input.getChildRefs(), ChildRef.TO_URI);
   ResolvedContent children = contentResolver.findByCanonicalUris(childUris);
   return Iterables.filter(children.getAllResolvedResults(), Item.class);
 }