/**
* @param searchResults: list of search results to be written into file
* @param query: helps at creating the file name
* @return file
*/
public static File writeVideoInfoToFile(List<SearchResult> searchResults, String query) {
if (searchResults.isEmpty()) {
System.out.println(" No video info to write to file for query:" + query);
return new File("");
}
try {
PrintWriter writer = new PrintWriter(getFileName(query), CSN);
for (SearchResult videoResult : searchResults) {
writer.println(videoResult.getSnippet().getTitle());
writer.println(videoResult.getSnippet().getDescription());
}
writer.close();
} catch (FileNotFoundException | UnsupportedEncodingException e) {
e.printStackTrace();
}
return new File(getFileName(query));
}
public List<VideoItem> search(String keywords) {
query.setQ(keywords);
try {
SearchListResponse response = query.execute();
List<SearchResult> results = response.getItems();
List<VideoItem> items = new ArrayList<VideoItem>();
for (SearchResult result : results) {
VideoItem item = new VideoItem();
item.setTitle(result.getSnippet().getTitle());
try {
item.setPublishedDate(
new SimpleDateFormat("MM/dd/yyy")
.format(
(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.parse(result.getSnippet().getPublishedAt().toString()))));
} catch (ParseException e) {
Log.d(TAG, "Could parse the date: " + e);
}
item.setDescription(result.getSnippet().getDescription());
item.setThumbnailURL(result.getSnippet().getThumbnails().getDefault().getUrl());
item.setId(result.getId().getVideoId());
items.add(item);
}
return items;
} catch (IOException e) {
Log.d(TAG, "Could not search: " + e);
return null;
}
}
public ArrayList<YoutubeVideo> loadMore(String offset, String keyword, int max) {
try {
query = youtube.search().list("id,snippet");
query.setKey(this.KEY);
query.setOrder("date");
query.setType("video");
query.setChannelId(this.channel_id);
if (keyword != null) {
query.setQ(keyword);
}
query.setPageToken(offset);
query.setMaxResults((long) max);
query.setFields(queryParams);
} catch (IOException e) {
Log.d("MORE", "Could not load more: " + e.getMessage());
}
SearchListResponse response = null;
try {
response = query.execute();
} catch (GoogleJsonResponseException e) {
switch (e.getStatusCode()) {
case 403:
this.KEY = this.context.getString(R.string.api_key2);
query.setKey(this.KEY);
try {
response = query.execute();
} catch (IOException e1) {
Log.d("MORE-KEY2", "Could not initialize: " + e1.getMessage());
}
}
} catch (IOException e) {
Log.d("MORE", "Could not execute more: " + e.getMessage());
return null;
}
List<SearchResult> results = response.getItems();
ArrayList<YoutubeVideo> items = new ArrayList<YoutubeVideo>();
for (SearchResult result : results) {
YoutubeVideo item = new YoutubeVideo();
item.setTitle(result.getSnippet().getTitle());
item.setDescription(result.getSnippet().getDescription());
item.setThumbnailURL(result.getSnippet().getThumbnails().getMedium().getUrl());
item.setId(result.getId().getVideoId());
item.setNextPageToken(response.getNextPageToken());
if (keyword != null) {
item.setKeyword(keyword);
}
try {
item.setDate(getDateFormat().parse(result.getSnippet().getPublishedAt().toString()));
} catch (ParseException e) {
e.printStackTrace();
}
item.setChannel_id(this.channel_id);
items.add(item);
}
return items;
}
/*
* Prints out all SearchResults in the Iterator. Each printed line includes title, id, and
* thumbnail.
*
* @param iteratorSearchResults Iterator of SearchResults to print
*
* @param query Search query (String)
*/
private static void prettyPrint(Iterator<SearchResult> iteratorSearchResults, String query) {
System.out.println("\n=============================================================");
System.out.println(
" First " + NUMBER_OF_VIDEOS_RETURNED + " videos for search on \"" + query + "\".");
System.out.println("=============================================================\n");
if (!iteratorSearchResults.hasNext()) {
System.out.println(" There aren't any results for your query.");
}
while (iteratorSearchResults.hasNext()) {
SearchResult singleVideo = iteratorSearchResults.next();
ResourceId rId = singleVideo.getId();
// Double checks the kind is video.
if (rId.getKind().equals("youtube#video"))
// Thumbnail thumbnail = singleVideo.getSnippet().getThumbnails().get("default");
System.out.println(" Video Id: " + rId.getVideoId());
System.out.println(" Title: " + singleVideo.getSnippet().getTitle());
// System.out.println(" Thumbnail: " + thumbnail.getUrl());
System.out.println("\n-------------------------------------------------------------\n");
// download it
String link =
"http://youtubeinmp3.com/fetch/?video=https://www.youtube.com/watch?v="
+ rId.getVideoId();
System.out.println("to download: " + link);
WebDownload wd = new WebDownload();
try {
wd.downloadMp3FromYoutube("download2.mp3", link);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@SuppressWarnings("deprecation")
@Override
public Boolean crawl(RssEntityBean site) throws CrawlerException {
log.info("start YouTube crawl channel id = " + site.url);
Boolean updated = false;
YouTube youtube =
new YouTube.Builder(
HTTP_TRANSPORT,
JSON_FACTORY,
new HttpRequestInitializer() {
public void initialize(HttpRequest request) throws IOException {}
})
.setApplicationName("youtube-cmdline-search-sample")
.build();
try {
YouTube.Search.List search = youtube.search().list("id,snippet");
String apiKey = Conf.getValue("youtube.appkey");
search.setKey(apiKey);
search.setType("video");
search.setOrder("date");
search.setMaxResults(maxResult);
search.setChannelId(site.url);
SearchListResponse searchResponse = search.execute();
List<SearchResult> searchResultList = searchResponse.getItems();
log.info("search response video count = " + searchResultList.size());
for (SearchResult result : searchResultList) {
SearchResultSnippet snippet = result.getSnippet();
DateTime publishDate = snippet.getPublishedAt();
DateTime lastCrawlDate = null;
if (site.lastCrawl != null) {
lastCrawlDate = new DateTime(site.lastCrawl);
}
if ((lastCrawlDate == null) || (publishDate.getValue() > lastCrawlDate.getValue())) {
YouTube.Videos.List videolist = youtube.videos().list("id,snippet");
videolist.setId(result.getId().getVideoId());
videolist.setKey(apiKey);
VideoListResponse videoListResponse = videolist.execute();
List<Video> videos = videoListResponse.getItems();
if (videos.size() != 1) {
log.warn("video not found : " + videolist.getId());
continue;
}
Video video = videos.get(0);
ArticleEntityBean article = new ArticleEntityBean();
article.link = video.getId();
article.title = snippet.getTitle();
article.description = snippet.getDescription();
article.imageUrl = snippet.getThumbnails().getDefault().getUrl();
article.url = site.url;
article.createdAt = snippet.getPublishedAt();
String fixHour = Conf.getValue("fix_time");
Date now = new Date();
Date fixDate =
new Date(now.getYear(), now.getMonth(), now.getDate(), Integer.parseInt(fixHour), 0);
if (now.getTime() > fixDate.getTime()) {
article.publishedAt = tomorrow;
} else {
article.publishedAt = today;
}
article.site = site.site;
article.type = site.type;
article.tags = site.defaultTag;
log.info("save article video id = " + article.link + " / video title = " + article.title);
article.save();
updated = true;
}
}
} catch (IOException e) {
log.warn("youtube api error : " + e.getMessage());
throw new CrawlerException(e);
}
log.info("end YouTube crawl");
return updated;
}
/**
* Initializes YouTube object to search for videos on YouTube (Youtube.Search.List). The program
* then prints the names and thumbnails of each of the videos (only first 50 videos).
*/
public String videoIdFromQueryTerm(String queryTerm) {
// Read the developer key from youtube.properties
String videoId = null;
Properties properties = new Properties();
// try {
// InputStream in = Search.class.getResourceAsStream("/" + PROPERTIES_FILENAME);
// properties.load(in);
//
// } catch (IOException e) {
// System.err.println("There was an error reading " + PROPERTIES_FILENAME + ": " +
// e.getCause()
// + " : " + e.getMessage());
// System.exit(1);
// }
try {
/*
* The YouTube object is used to make all API requests. The last argument is required, but
* because we don't need anything initialized when the HttpRequest is initialized, we override
* the interface and provide a no-op function.
*/
youtube =
new YouTube.Builder(
HTTP_TRANSPORT,
JSON_FACTORY,
new HttpRequestInitializer() {
public void initialize(HttpRequest request) throws IOException {}
})
.setApplicationName("youtube-cmdline-search-sample")
.build();
// Get query term from user.
// String queryTerm = getInputQuery();
YouTube.Search.List search = youtube.search().list("id,snippet");
/*
* It is important to set your developer key from the Google Developer Console for
* non-authenticated requests (found under the API Access tab at this link:
* code.google.com/apis/). This is good practice and increased your quota.
*/
String apiKey = "AIzaSyB--WhmXYTrUlitFPWtiPZu8V1w_iPRCbw";
search.setKey(apiKey);
search.setQ(queryTerm);
/*
* We are only searching for videos (not playlists or channels). If we were searching for
* more, we would add them as a string like this: "video,playlist,channel".
*/
search.setType("video");
/*
* This method reduces the info returned to only the fields we need and makes calls more
* efficient.
*/
search.setFields("items(id/kind,id/videoId,snippet/title,snippet/thumbnails/default/url)");
search.setMaxResults(NUMBER_OF_VIDEOS_RETURNED);
SearchListResponse searchResponse = search.execute();
List<SearchResult> searchResultList = searchResponse.getItems();
SearchResult searchResult = searchResultList.get(0);
ResourceId rId = searchResult.getId();
videoId = rId.getVideoId();
// if (searchResultList != null) {
// prettyPrint(searchResultList.iterator(), queryTerm);
// }
} catch (GoogleJsonResponseException e) {
System.err.println(
"There was a service error: "
+ e.getDetails().getCode()
+ " : "
+ e.getDetails().getMessage());
} catch (IOException e) {
System.err.println("There was an IO error: " + e.getCause() + " : " + e.getMessage());
} catch (Throwable t) {
t.printStackTrace();
}
return videoId;
}