public JSONArray Search(String query, String sort) {
SolrServer server;
SolrDocumentList docs = null;
String Date_format = null; // for Date格式轉換
JSONArray jsonArray = new JSONArray();
try {
server = new HttpSolrServer(url);
// add
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery(query);
solrQuery.setStart(_start);
solrQuery.setRows(_nbDocuments);
solrQuery.setRequestHandler("query");
solrQuery.set("fl", "*,score"); // 設定fl參數，指定要傳回哪個field的資料，這裡設定所有field與score
if (_fq1 != "") {
solrQuery.addFilterQuery("ProductName:" + _fq1);
}
if (_fq2 != "") {
solrQuery.addFilterQuery("publishedDate:" + _fq2);
}
if (sort != null) {
solrQuery.addSortField(sort, ORDER.asc);
}
solrQuery.setRequestHandler("/browse");
response = server.query(solrQuery);
docs = response.getResults();
if (docs != null) {
System.out.println(
docs.getNumFound() + " documents found, " + docs.size() + " returned : ");
setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
JSONObject jsonObject = new JSONObject();
for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) {
Entry<String, Object> entry = it2.next();
if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串，方便前端呈現
Date_format = entry.getValue().toString();
jsonObject.put(entry.getKey(), Date_format);
} else {
// 一般情況
jsonObject.put(entry.getKey(), entry.getValue());
}
}
System.out.print("\n");
// 將總共找到幾筆資料存在jsonarray的最後面傳給前端
jsonObject.put("TotalResultFound", docs.getNumFound());
jsonArray.add(jsonObject);
}
}
} catch (SolrServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return jsonArray;
}
@Test(dependsOnMethods = "testAddDocument")
public void testBasicQuery() throws SolrServerException {
SolrServer server = solrServerFactory.getServer();
SolrParams params = new SolrQuery("*:*");
SolrDocumentList results = server.query(params).getResults();
assertTrue(results.getNumFound() > 0, "didn't return any results");
}
protected List<Vector> processQueryResponse(QueryResponse resp) {
NamedList<Object> response = resp.getResponse();
NamedList<Object> termVectorsNL = (NamedList<Object>) response.get("termVectors");
if (termVectorsNL == null)
throw new RuntimeException(
"No termVectors in response! "
+ "Please check your query to make sure it is requesting term vector information from Solr correctly.");
List<Vector> termVectors = new ArrayList<Vector>(termVectorsNL.size());
Iterator<Map.Entry<String, Object>> iter = termVectorsNL.iterator();
while (iter.hasNext()) {
Map.Entry<String, Object> next = iter.next();
String nextKey = next.getKey();
Object nextValue = next.getValue();
if (nextValue instanceof NamedList) {
NamedList nextList = (NamedList) nextValue;
Object fieldTerms = nextList.get(field);
if (fieldTerms != null && fieldTerms instanceof NamedList) {
termVectors.add(
SolrTermVector.newInstance(nextKey, hashingTF, (NamedList<Object>) fieldTerms));
}
}
}
SolrDocumentList docs = resp.getResults();
totalDocs = docs.getNumFound();
return termVectors;
}
public SearchResponse search(SearchRequest request) {
SolrQuery query = new SolrQuery();
query.setRows(request.getLimit());
query.setStart(request.getOffset() * request.getLimit());
BooleanQuery aggregate = new BooleanQuery();
if (!StringUtils.isBlank(request.getText())) {
TermQuery termQuery = new TermQuery(new Term("", request.getText()));
aggregate.add(termQuery, BooleanClause.Occur.SHOULD);
}
if (!StringUtils.isBlank(request.getEventId())) {
aggregate.add(
new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST);
}
query.setQuery(aggregate.toString());
log.info("QUERY IS: " + query.toString());
try {
QueryResponse queryResponse = solrServer.query(query);
log.info("RESPONSE FROM QUERY WAS: " + queryResponse);
SolrDocumentList results = queryResponse.getResults();
log.info("RESULTS WAS: " + results);
ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>();
for (SolrDocument result : results) {
hits.add(
new SearchResponse.Hit(
ObjectType.session, String.valueOf(result.getFieldValue("id")), null));
}
return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits);
} catch (SolrServerException e) {
e.printStackTrace();
}
return null;
}
/**
* Returns SearchResultsSlim for a given SolrDocumentList. A "slimmer" results object without the
* "items" wrapper element is created for better transform to json.
*
* @param doc solr document list to build results
* @return the SearchResultsSlim object for this solr result
* @see SearchResultsSlim
*/
private SearchResultsSlim buildSlimResults(SolrDocumentList docs) {
SearchResultsSlim results = new SearchResultsSlim();
Pagination pagination = new Pagination();
pagination.setNumFound(docs.getNumFound());
pagination.setStart(docs.getStart());
pagination.setRows(limit);
// List<ModsType> modsTypes = new ArrayList<ModsType>();
List<Item> items = new ArrayList<Item>();
for (final SolrDocument doc : docs) {
Item item = new Item();
ModsType modsType = null;
try {
modsType = (new ItemDAO()).getModsType(doc);
} catch (JAXBException je) {
log.error(je.getMessage());
je.printStackTrace();
}
item.setModsType(modsType);
items.add(item);
}
results.setItems(items);
results.setPagination(pagination);
if (facet != null) results.setFacet(facet);
return results;
}
@Test(dependsOnMethods = "testBasicQuery", dataProvider = "documentMap")
public void testIdQuery(Map<String, String[]> document) throws SolrServerException {
String id = document.get("id")[0];
SolrServer server = solrServerFactory.getServer();
SolrParams params = new SolrQuery("id:" + id);
SolrDocumentList results = server.query(params).getResults();
assertEquals(results.getNumFound(), 1, "didn't find article by id");
}
public void writeSolrDocumentList(SolrDocumentList docs) throws IOException {
writeTag(SOLRDOCLST);
List<Number> l = new ArrayList<>(3);
l.add(docs.getNumFound());
l.add(docs.getStart());
l.add(docs.getMaxScore());
writeArray(l);
writeArray(docs);
}
@Override
public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
productsToFilter = new ArrayList<String>();
String queryString = "*:*";
String sortCriteria = "user_count_purchased desc";
ModifiableSolrParams solrParams = new ModifiableSolrParams();
QueryResponse response = null;
RecommendResponse searchResponse = new RecommendResponse();
try {
String filterQueryString =
RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter);
if (query.getProductIds() != null && query.getProductIds().size() > 0) {
productsToFilter.addAll(query.getProductIds());
}
if (query.getUser() != null) {
if (alreadyBoughtProducts != null) {
productsToFilter.addAll(alreadyBoughtProducts);
}
}
if (productsToFilter != null && productsToFilter.size() > 0) {
if (filterQueryString.length() > 0) {
filterQueryString += " OR ";
}
filterQueryString +=
RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter);
}
solrParams.set("q", queryString);
solrParams.set("fq", filterQueryString);
solrParams.set("sort", sortCriteria);
solrParams.set("rows", maxReuslts);
response =
SolrServiceContainer.getInstance()
.getRecommendService()
.getSolrServer()
.query(solrParams);
// fill response object
List<String> extractedRecommendations =
RecommendationQueryUtils.extractRecommendationIds(
response.getBeans(CustomerAction.class));
searchResponse.setResultItems(extractedRecommendations);
searchResponse.setElapsedTime(response.getElapsedTime());
SolrDocumentList docResults = response.getResults();
searchResponse.setNumFound(docResults.getNumFound());
} catch (SolrServerException e) {
e.printStackTrace();
searchResponse.setNumFound(0);
searchResponse.setResultItems(new ArrayList<String>());
searchResponse.setElapsedTime(-1);
}
return searchResponse;
}
@Override
public void writeSolrDocumentList(
String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
boolean includeScore = false;
if (fields != null) {
includeScore = fields.contains("score");
if (fields.size() == 0 || (fields.size() == 1 && includeScore) || fields.contains("*")) {
fields = null; // null means return all stored fields
}
}
int sz = docs.size();
writeMapOpener(includeScore ? 4 : 3);
incLevel();
writeKey("numFound", false);
writeLong(null, docs.getNumFound());
writeMapSeparator();
writeKey("start", false);
writeLong(null, docs.getStart());
if (includeScore && docs.getMaxScore() != null) {
writeMapSeparator();
writeKey("maxScore", false);
writeFloat(null, docs.getMaxScore());
}
writeMapSeparator();
// indent();
writeKey("docs", false);
writeArrayOpener(sz);
incLevel();
boolean first = true;
SolrIndexSearcher searcher = req.getSearcher();
for (SolrDocument doc : docs) {
if (first) {
first = false;
} else {
writeArraySeparator();
}
indent();
writeSolrDocument(null, doc, fields, otherFields);
}
decLevel();
writeArrayCloser();
if (otherFields != null) {
writeMap(null, otherFields, true, false);
}
decLevel();
indent();
writeMapCloser();
}
@Override
public <T> ScoredPage<T> queryForPage(Query query, Class<T> clazz) {
Assert.notNull(query, "Query must not be 'null'.");
Assert.notNull(clazz, "Target class must not be 'null'.");
QueryResponse response = query(query);
List<T> beans = convertQueryResponseToBeans(response, clazz);
SolrDocumentList results = response.getResults();
return new SolrResultPage<T>(
beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
}
@Test
public void testGetIdsWithParams() throws Exception {
SolrDocumentList rsp =
getSolrClient().getById(Arrays.asList("0", "1", "2"), params(CommonParams.FL, "id"));
assertEquals(2, rsp.getNumFound());
assertEquals("1", rsp.get(0).get("id"));
assertNull("This field should have been removed from the response.", rsp.get(0).get("term_s"));
assertNull("This field should have been removed from the response.", rsp.get(0).get("term2_s"));
assertEquals("2", rsp.get(1).get("id"));
assertNull("This field should have been removed from the response.", rsp.get(1).get("term_s"));
assertNull("This field should have been removed from the response.", rsp.get(1).get("term2_s"));
}
/** SRN-96 */
@Test
public void testASCIIFoldingExpansion() throws IOException, SolrServerException {
this.addJsonString("1", " { \"value\" : \"cafe\" } ");
this.addJsonString("2", " { \"value\" : \"café\" } ");
SolrQuery query = new SolrQuery();
query.setQuery("cafe");
query.setRequestHandler("keyword");
query.setIncludeScore(true);
// should match the two documents, with same score
QueryResponse response = getWrapper().getServer().query(query);
SolrDocumentList docList = response.getResults();
assertEquals(2, docList.getNumFound());
float score1 = (Float) docList.get(0).getFieldValue("score");
float score2 = (Float) docList.get(1).getFieldValue("score");
Assert.assertTrue("Score should be identical", score1 == score2);
// should match the two documents, but should assign different score
// id2 should receive better score than id1
query = new SolrQuery();
query.setQuery("café");
query.setRequestHandler("keyword");
query.setIncludeScore(true);
response = getWrapper().getServer().query(query);
docList = response.getResults();
assertEquals(2, docList.getNumFound());
if (docList.get(0).getFieldValue("url").equals("id1")) {
score1 = (Float) docList.get(0).getFieldValue("score");
score2 = (Float) docList.get(1).getFieldValue("score");
} else {
score2 = (Float) docList.get(0).getFieldValue("score");
score1 = (Float) docList.get(1).getFieldValue("score");
}
Assert.assertTrue("id2 should get higher score than id1", score1 < score2);
}
// 查询
@Test
public void testSolrForSearch() throws Exception {
SolrQuery params = new SolrQuery();
params.set("q", "*:*");
QueryResponse response = solrServer.query(params);
// 结果集
SolrDocumentList docs = response.getResults();
long count = docs.getNumFound();
System.out.println("结果集有 ：" + count);
for (SolrDocument doc : docs) {
String id = (String) doc.get("id");
String name = (String) doc.get("name");
System.out.println("id :" + id + "---" + "name :" + name);
}
}
@Override
public <T> HighlightPage<T> queryForHighlightPage(HighlightQuery query, Class<T> clazz) {
Assert.notNull(query, "Query must not be 'null'.");
Assert.notNull(clazz, "Target class must not be 'null'.");
QueryResponse response = query(query);
List<T> beans = convertQueryResponseToBeans(response, clazz);
SolrDocumentList results = response.getResults();
SolrResultPage<T> page =
new SolrResultPage<T>(
beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
ResultHelper.convertAndAddHighlightQueryResponseToResultPage(response, page);
return page;
}
@Test
public void testGetIds() throws Exception {
SolrDocumentList rsp = getSolrClient().getById(Arrays.asList("0", "1", "2", "3", "4"));
assertEquals(3, rsp.getNumFound());
assertEquals("1", rsp.get(0).get("id"));
assertEquals("Microsoft", rsp.get(0).get("term_s"));
assertEquals("MSFT", rsp.get(0).get("term2_s"));
assertEquals("2", rsp.get(1).get("id"));
assertEquals("Apple", rsp.get(1).get("term_s"));
assertEquals("AAPL", rsp.get(1).get("term2_s"));
assertEquals("3", rsp.get(2).get("id"));
assertEquals("Yahoo", rsp.get(2).get("term_s"));
assertEquals("YHOO", rsp.get(2).get("term2_s"));
}
@Override
public <T> FacetPage<T> queryForFacetPage(FacetQuery query, Class<T> clazz) {
Assert.notNull(query, "Query must not be 'null'.");
Assert.notNull(clazz, "Target class must not be 'null'.");
QueryResponse response = query(query);
List<T> beans = convertQueryResponseToBeans(response, clazz);
SolrDocumentList results = response.getResults();
SolrResultPage<T> page =
new SolrResultPage<T>(
beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
page.addAllFacetFieldResultPages(
ResultHelper.convertFacetQueryResponseToFacetPageMap(query, response));
page.addAllFacetPivotFieldResult(
ResultHelper.convertFacetQueryResponseToFacetPivotMap(query, response));
page.setFacetQueryResultPage(
ResultHelper.convertFacetQueryResponseToFacetQueryResult(query, response));
return page;
}
private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest)
throws SolrServerException, IOException {
// verify the join with fromIndex works
final String fromQ = "match_s:c^2";
CloudSolrClient client = cluster.getSolrClient();
{
final String joinQ =
"{!join "
+ anyScoreMode(isScoresTest)
+ "from=join_s fromIndex="
+ fromColl
+ " to=join_s}"
+ fromQ;
QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
QueryResponse rsp = new QueryResponse(client.request(qr), client);
SolrDocumentList hits = rsp.getResults();
assertTrue("Expected 1 doc, got " + hits, hits.getNumFound() == 1);
SolrDocument doc = hits.get(0);
assertEquals(toDocId, doc.getFirstValue("id"));
assertEquals("b", doc.getFirstValue("get_s"));
assertScore(isScoresTest, doc);
}
// negative test before creating an alias
checkAbsentFromIndex(fromColl, toColl, isScoresTest);
// create an alias for the fromIndex and then query through the alias
String alias = fromColl + "Alias";
CollectionAdminRequest.CreateAlias request =
CollectionAdminRequest.createAlias(alias, fromColl);
request.process(client);
{
final String joinQ =
"{!join "
+ anyScoreMode(isScoresTest)
+ "from=join_s fromIndex="
+ alias
+ " to=join_s}"
+ fromQ;
final QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
final QueryResponse rsp = new QueryResponse(client.request(qr), client);
final SolrDocumentList hits = rsp.getResults();
assertTrue("Expected 1 doc", hits.getNumFound() == 1);
SolrDocument doc = hits.get(0);
assertEquals(toDocId, doc.getFirstValue("id"));
assertEquals("b", doc.getFirstValue("get_s"));
assertScore(isScoresTest, doc);
}
// negative test after creating an alias
checkAbsentFromIndex(fromColl, toColl, isScoresTest);
{
// verify join doesn't work if no match in the "from" index
final String joinQ =
"{!join "
+ (anyScoreMode(isScoresTest))
+ "from=join_s fromIndex="
+ fromColl
+ " to=join_s}match_s:d";
final QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
final QueryResponse rsp = new QueryResponse(client.request(qr), client);
final SolrDocumentList hits = rsp.getResults();
assertTrue("Expected no hits", hits.getNumFound() == 0);
}
}
/**
* Sends an email to the given e-person with details of new items in the given dspace object (MUST
* be a community or a collection), items that appeared yesterday. No e-mail is sent if there
* aren't any new items in any of the dspace objects.
*
* @param context DSpace context object
* @param eperson eperson to send to
* @param rpkeys List of DSpace Objects
* @param test
* @throws SearchServiceException
*/
public static void sendEmail(
Researcher researcher,
Context context,
EPerson eperson,
List<String> rpkeys,
boolean test,
List<String> relationFields)
throws IOException, MessagingException, SQLException, SearchServiceException {
CrisSearchService searchService = researcher.getCrisSearchService();
// Get a resource bundle according to the eperson language preferences
Locale supportedLocale = I18nUtil.getEPersonLocale(eperson);
StringBuffer emailText = new StringBuffer();
boolean isFirst = true;
for (String rpkey : rpkeys) {
SolrQuery query = new SolrQuery();
query.setFields("search.resourceid");
query.addFilterQuery(
"{!field f=search.resourcetype}" + Constants.ITEM, "{!field f=inarchive}true");
for (String tmpRelations : relationFields) {
String fq = "{!field f=" + tmpRelations + "}" + rpkey;
query.addFilterQuery(fq);
}
query.setRows(Integer.MAX_VALUE);
if (ConfigurationManager.getBooleanProperty("eperson.subscription.onlynew", false)) {
// get only the items archived yesterday
query.setQuery("dateaccessioned:(NOW/DAY-1DAY)");
} else {
// get all item modified yesterday but not published the day
// before
// and all the item modified today and archived yesterday
query.setQuery(
"(item.lastmodified:(NOW/DAY-1DAY) AND dateaccessioned:(NOW/DAY-1DAY)) OR ((item.lastmodified:(NOW/DAY) AND dateaccessioned:(NOW/DAY-1DAY)))");
}
QueryResponse qResponse = searchService.search(query);
SolrDocumentList results = qResponse.getResults();
// Only add to buffer if there are new items
if (results.getNumFound() > 0) {
if (!isFirst) {
emailText.append("\n---------------------------------------\n");
} else {
isFirst = false;
}
emailText
.append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.new-items", supportedLocale))
.append(" ")
.append(rpkey)
.append(": ")
.append(results.getNumFound())
.append("\n\n");
for (SolrDocument solrDoc : results) {
Item item = Item.find(context, (Integer) solrDoc.getFieldValue("search.resourceid"));
DCValue[] titles = item.getDC("title", null, Item.ANY);
emailText
.append(" ")
.append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.title", supportedLocale))
.append(" ");
if (titles.length > 0) {
emailText.append(titles[0].value);
} else {
emailText.append(
I18nUtil.getMessage("org.dspace.eperson.Subscribe.untitled", supportedLocale));
}
DCValue[] authors = item.getDC("contributor", Item.ANY, Item.ANY);
if (authors.length > 0) {
emailText
.append("\n ")
.append(
I18nUtil.getMessage("org.dspace.eperson.Subscribe.authors", supportedLocale))
.append(" ")
.append(authors[0].value);
for (int k = 1; k < authors.length; k++) {
emailText.append("\n ").append(authors[k].value);
}
}
emailText
.append("\n ")
.append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.id", supportedLocale))
.append(" ")
.append(HandleManager.getCanonicalForm(item.getHandle()))
.append("\n\n");
context.removeCached(item, item.getID());
}
}
}
// Send an e-mail if there were any new items
if (emailText.length() > 0) {
if (test) {
log.info(LogManager.getHeader(context, "subscription:", "eperson=" + eperson.getEmail()));
log.info(LogManager.getHeader(context, "subscription:", "text=" + emailText.toString()));
} else {
Email email =
ConfigurationManager.getEmail(
I18nUtil.getEmailFilename(supportedLocale, "subscription"));
email.addRecipient(eperson.getEmail());
email.addArgument(emailText.toString());
email.send();
log.info(
LogManager.getHeader(context, "sent_subscription", "eperson_id=" + eperson.getID()));
}
}
}
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType(XML_RESPONSE_HEADER); // Talkback happens in XML form.
response.setCharacterEncoding("UTF-8"); // Unicode++
request.setCharacterEncoding("UTF-8");
PrintWriter out = null; // The talkback buffer.
// handle startrecord
Integer startRecord = 0;
if (!(request.getParameter("startRecord") == null)) {
try {
startRecord = Integer.parseInt(request.getParameter("startRecord")) - 1;
} catch (NumberFormatException e) {
startRecord = 0;
}
}
// maximumrecords
Integer maximumRecords = Integer.parseInt(this.config.getProperty("default_maximumRecords"));
if (!(request.getParameter("maximumRecords") == null)) {
maximumRecords = Integer.parseInt(request.getParameter("maximumRecords"));
}
// operation
String operation = request.getParameter("operation");
// x_collection
String x_collection = request.getParameter("x-collection");
if (x_collection == null) x_collection = this.config.getProperty("default_x_collection");
if (x_collection == null) operation = null;
// sortkeys
String sortKeys = request.getParameter("sortKeys");
// sortorder
String sortOrder = request.getParameter("sortOrder");
// recordschema
String recordSchema = request.getParameter("recordSchema");
if (recordSchema == null) recordSchema = "dc";
if (recordSchema.equalsIgnoreCase("dcx")) {
recordSchema = "dcx";
}
if (recordSchema.equalsIgnoreCase("solr")) {
recordSchema = "solr";
}
// query request
String query = request.getParameter("query");
String q = request.getParameter("q");
// who is requestor ?
String remote_ip = request.getHeader("X-FORWARDED-FOR");
if (remote_ip == null) {
remote_ip = request.getRemoteAddr().trim();
} else {
remote_ip = request.getHeader("X-FORWARDED-FOR");
}
// handle debug
Boolean debug = Boolean.parseBoolean(request.getParameter("debug"));
if (!debug) {
out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
}
// handle query
if ((query == null) && (q != null)) {
query = q;
} else {
if ((query != null) && (q == null)) {
q = query;
} else {
operation = null;
}
}
// handle operation
if (operation == null) {
if (query != null) {
operation = "searchRetrieve";
} else {
operation = "explain";
}
}
// searchRetrieve
if (operation.equalsIgnoreCase("searchRetrieve")) {
if (query == null) {
operation = "explain";
log.debug(operation + ":" + query);
}
}
// start talking back.
String[] sq = {""};
String solrquery = "";
// facet
String facet = null;
List<FacetField> fct = null;
if (request.getParameter("facet") != null) {
facet = request.getParameter("facet");
log.debug("facet : " + facet);
}
if (operation == null) {
operation = "searchretrieve";
} else { // explain response
if (operation.equalsIgnoreCase("explain")) {
log.debug("operation = explain");
out.write("<srw:explainResponse xmlns:srw=\"http://www.loc.gov/zing/srw/\">");
out.write("</srw:explainResponse>");
} else { // DEBUG routine
operation = "searchretrieve";
String triplequery = null;
if (query.matches(".*?\\[.+?\\].*?")) { // New symantic syntax
triplequery = symantic_query(query);
query = query.split("\\[")[0] + " " + triplequery;
log.fatal(triplequery);
solrquery = CQLtoLucene.translate(query, log, config);
} else {
solrquery = CQLtoLucene.translate(query, log, config);
}
log.debug(solrquery);
if (debug == true) {
response.setContentType(HTML_RESPONSE_HEADER);
out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
out.write("<html><body>\n\n");
out.write("'" + remote_ip + "'<br>\n");
out.write("<form action='http://www.kbresearch.nl/kbSRU'>");
out.write("<input type=text name=q value='" + query + "' size=120>");
out.write("<input type=hidden name=debug value=True>");
out.write("<input type=submit>");
out.write("<table border=1><tr><td>");
out.write("q</td><td>" + query + "</td></tr><tr>");
out.write("<td>query out</td><td>" + URLDecoder.decode(solrquery) + "</td></tr>");
out.write(
"<tr><td>SOLR_URL</td><td> <a href='"
+ this.config.getProperty(
"collection." + x_collection.toLowerCase() + ".solr_baseurl")
+ "/?q="
+ solrquery
+ "'>"
+ this.config.getProperty(
"collection." + x_collection.toLowerCase() + ".solr_baseurl")
+ "/select/?q="
+ solrquery
+ "</a><br>"
+ this.config.getProperty("solr_url")
+ solrquery
+ "</td></tr>");
out.write(
"<b>SOLR_QUERY</b> : <BR> <iframe width=900 height=400 src='"
+ this.config.getProperty(
"collection." + x_collection.toLowerCase() + ".solr_baseurl")
+ "/../?q="
+ solrquery
+ "'></iframe><BR>");
out.write(
"<b>SRU_QUERY</b> : <BR> <a href="
+ this.config.getProperty("baseurl")
+ "?q="
+ query
+ "'>"
+ this.config.getProperty("baseurl")
+ "?q="
+ query
+ "</a><br><iframe width=901 height=400 src='http://www.kbresearch.nl/kbSRU/?q="
+ query
+ "'></iframe><BR>");
out.write(
"<br><b>JSRU_QUERY</b> : <BR><a href='http://jsru.kb.nl/sru/?query="
+ query
+ "&x-collection="
+ x_collection
+ "'>http://jsru.kb.nl/sru/?query="
+ query
+ "&x-collection=GGC</a><br><iframe width=900 height=400 src='http://jsru.kb.nl/sru/?query="
+ query
+ "&x-collection=GGC'></iframe>");
} else { // XML SearchRetrieve response
String url =
this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl");
String buffer = "";
CommonsHttpSolrServer server = null;
server = new CommonsHttpSolrServer(url);
log.fatal("URSING " + url);
server.setParser(new XMLResponseParser());
int numfound = 0;
try {
SolrQuery do_query = new SolrQuery();
do_query.setQuery(solrquery);
do_query.setRows(maximumRecords);
do_query.setStart(startRecord);
if ((sortKeys != null) && (sortKeys.length() > 1)) {
if (sortOrder != null) {
if (sortOrder.equals("asc")) {
do_query.setSortField(sortKeys, SolrQuery.ORDER.asc);
}
if (sortOrder.equals("desc")) {
do_query.setSortField(sortKeys, SolrQuery.ORDER.desc);
}
} else {
for (String str : sortKeys.trim().split(",")) {
str = str.trim();
if (str.length() > 1) {
if (str.equals("date")) {
do_query.setSortField("date_date", SolrQuery.ORDER.desc);
log.debug("SORTORDERDEBUG | DATE! " + str + " | ");
break;
} else {
do_query.setSortField(str + "_str", SolrQuery.ORDER.asc);
log.debug("SORTORDERDEBUG | " + str + " | ");
break;
}
}
}
}
}
if (facet != null) {
if (facet.indexOf(",") > 1) {
for (String str : facet.split(",")) {
if (str.indexOf("date") > 1) {
do_query.addFacetField(str);
} else {
do_query.addFacetField(str);
}
// do_query.setParam("facet.method", "enum");
}
// q.setFacetSort(false);
} else {
do_query.addFacetField(facet);
}
do_query.setFacet(true);
do_query.setFacetMinCount(1);
do_query.setFacetLimit(-1);
}
log.fatal(solrquery);
QueryResponse rsp = null;
boolean do_err = false;
boolean do_sugg = false;
SolrDocumentList sdl = null;
String diag = "";
StringBuffer suggest = new StringBuffer("");
String content = "1";
SolrQuery spellq = do_query;
try {
rsp = server.query(do_query);
} catch (SolrServerException e) {
String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", "0");
out.write(header);
diag = this.SRW_DIAG.replaceAll("\\$error", e.getMessage());
do_err = true;
rsp = null;
}
log.fatal("query done..");
if (!(do_err)) { // XML dc response
SolrDocumentList docs = rsp.getResults();
numfound = (int) docs.getNumFound();
int count = startRecord;
String header =
this.SRW_HEADER.replaceAll("\\$numberOfRecords", Integer.toString(numfound));
out.write(header);
out.write("<srw:records>");
Iterator<SolrDocument> iter = rsp.getResults().iterator();
while (iter.hasNext()) {
count += 1;
if (recordSchema.equalsIgnoreCase("dc")) {
SolrDocument resultDoc = iter.next();
content = (String) resultDoc.getFieldValue("id");
out.write("<srw:record>");
out.write("<srw:recordPacking>xml</srw:recordPacking>");
out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
out.write(
"<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" xmlns:facets=\"info:srw/extension/4/facets\" >");
StringBuffer result = new StringBuffer("");
construct_lucene_dc(result, resultDoc);
out.write(result.toString());
out.write("</srw:recordData>");
out.write(
"<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
out.write("</srw:record>");
}
if (recordSchema.equalsIgnoreCase("solr")) {
SolrDocument resultDoc = iter.next();
content = (String) resultDoc.getFieldValue("id");
out.write("<srw:record>");
out.write("<srw:recordPacking>xml</srw:recordPacking>");
out.write("<srw:recordSchema>info:srw/schema/1/solr</srw:recordSchema>");
out.write("<srw:recordData xmlns:expand=\"http://www.kbresearch.nl/expand\">");
StringBuffer result = new StringBuffer("");
construct_lucene_solr(result, resultDoc);
out.write(result.toString());
out.write("</srw:recordData>");
out.write(
"<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
out.write("</srw:record>");
}
if (recordSchema.equalsIgnoreCase("dcx")) { // XML dcx response
out.write("<srw:record>");
out.write("<srw:recordPacking>xml</srw:recordPacking>");
out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
out.write(
"<srw:recordData><srw_dc:dc xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/marc.relators/\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
SolrDocument resultDoc = iter.next();
content = (String) resultDoc.getFieldValue("id");
String dcx_data =
helpers.getOAIdcx(
"http://services.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
log);
if (x_collection.equalsIgnoreCase("ggc-thes")) {
dcx_data =
helpers.getOAIdcx(
"http://serviceso.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
log);
}
if (!(dcx_data.length() == 0)) {
out.write(dcx_data);
} else {
// Should not do this!!
out.write("<srw:record>");
out.write("<srw:recordPacking>xml</srw:recordPacking>");
out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
out.write(
"<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" >");
StringBuffer result = new StringBuffer("");
construct_lucene_dc(result, resultDoc);
out.write(result.toString());
out.write("</srw:recordData>");
out.write(
"<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
out.write("</srw:record>");
}
out.write("</srw_dc:dc>");
StringBuffer expand_data;
boolean expand = false;
if (content.startsWith("GGC-THES:AC:")) {
String tmp_content = "";
tmp_content = content.replaceFirst("GGC-THES:AC:", "");
log.fatal("calling get");
expand_data =
new StringBuffer(
helpers.getExpand(
"http://www.kbresearch.nl/general/lod_new/get/"
+ tmp_content
+ "?format=rdf",
log));
log.fatal("get finini");
if (expand_data.toString().length() > 4) {
out.write(
"<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
out.write(expand_data.toString());
expand = true;
}
} else {
expand_data =
new StringBuffer(
helpers.getExpand(
"http://www.kbresearch.nl/ANP.cgi?q=" + content, log));
if (expand_data.toString().length() > 0) {
if (!expand) {
out.write(
"<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
expand = true;
}
out.write(expand_data.toString());
}
}
if (expand) {
out.write("</srw_dc:expand>");
}
out.write("</srw:recordData>");
out.write(
"<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
out.write("</srw:record>");
}
}
}
if ((do_err) || (numfound == 0)) {
log.fatal("I haz suggestions");
try {
spellq.setParam("spellcheck", true);
spellq.setQueryType("/spell");
server = new CommonsHttpSolrServer(url);
rsp = server.query(spellq);
sdl = rsp.getResults();
SpellCheckResponse spell;
spell = rsp.getSpellCheckResponse();
List<SpellCheckResponse.Suggestion> suggestions = spell.getSuggestions();
if (suggestions.isEmpty() == false) {
suggest.append("<srw:extraResponseData>");
suggest.append("<suggestions>");
for (SpellCheckResponse.Suggestion sugg : suggestions) {
suggest.append("<suggestionfor>" + sugg.getToken() + "</suggestionfor>");
for (String item : sugg.getSuggestions()) {
suggest.append("<suggestion>" + item + "</suggestion>");
}
suggest.append("</suggestions>");
suggest.append("</srw:extraResponseData>");
}
do_sugg = true;
}
} catch (Exception e) {
rsp = null;
// log.fatal(e.toString());
}
;
}
;
if (!do_err) {
if (facet != null) {
try {
fct = rsp.getFacetFields();
out.write("<srw:facets>");
for (String str : facet.split(",")) {
out.write("<srw:facet>");
out.write("<srw:facetType>");
out.write(str);
out.write("</srw:facetType>");
for (FacetField f : fct) {
log.debug(f.getName());
// if (f.getName().equals(str+"_str") || (f.getName().equals(str+"_date")) ) {
List<FacetField.Count> facetEnties = f.getValues();
for (FacetField.Count fcount : facetEnties) {
out.write("<srw:facetValue>");
out.write("<srw:valueString>");
out.write(helpers.xmlEncode(fcount.getName()));
out.write("</srw:valueString>");
out.write("<srw:count>");
out.write(Double.toString(fcount.getCount()));
out.write("</srw:count>");
out.write("</srw:facetValue>");
// }
}
}
out.write("</srw:facet>");
}
out.write("</srw:facets>");
startRecord += 1;
} catch (Exception e) {
}
// log.fatal(e.toString()); }
}
} else {
out.write(diag);
}
out.write("</srw:records>"); // SearchRetrieve response footer
String footer = this.SRW_FOOTER.replaceAll("\\$query", helpers.xmlEncode(query));
footer = footer.replaceAll("\\$startRecord", (startRecord).toString());
footer = footer.replaceAll("\\$maximumRecords", maximumRecords.toString());
footer = footer.replaceAll("\\$recordSchema", recordSchema);
if (do_sugg) {
out.write(suggest.toString());
}
out.write(footer);
} catch (MalformedURLException e) {
out.write(e.getMessage());
} catch (IOException e) {
out.write("TO ERR is Human");
}
}
}
}
out.close();
}
/** Main method of this example. */
@Override
public void runExample(ExampleDriver driver) throws Exception {
long startMs = System.currentTimeMillis();
CommandLine cli = driver.getCommandLine();
// Size of index batch requests to Solr
// int batchSize = Integer.parseInt(cli.getOptionValue("batchSize", "500"));
// Get a connection to Solr cloud using Zookeeper
String zkHost = cli.getOptionValue("zkhost", ZK_HOST);
String collectionName = cli.getOptionValue("collection", COLLECTION);
int zkClientTimeout = Integer.parseInt(cli.getOptionValue("zkClientTimeout", "15000"));
CloudSolrServer solr = new CloudSolrServer(zkHost);
solr.setDefaultCollection(collectionName);
solr.setZkClientTimeout(zkClientTimeout);
solr.connect();
int numSent = 0;
int numSkipped = 0;
int lineNum = 0;
SolrInputDocument doc = null;
String line = null;
// read file line-by-line
BufferedReader reader = new BufferedReader(driver.readFile("log"));
driver.rememberCloseable(reader);
LogFormat fmt = LogFormat.valueOf(cli.getOptionValue("format", "solr"));
// process each sighting as a document
while ((line = reader.readLine()) != null) {
doc = parseNextDoc(line, ++lineNum, fmt);
if (doc != null) {
addDocWithRetry(solr, doc, 10);
++numSent;
} else {
++numSkipped;
continue;
}
if (lineNum % 1000 == 0) log.info(String.format("Processed %d lines.", lineNum));
}
// hard commit all docs sent
solr.commit(true, true);
float tookSecs = Math.round(((System.currentTimeMillis() - startMs) / 1000f) * 100f) / 100f;
log.info(
String.format(
"Sent %d log messages (skipped %d) took %f seconds", numSent, numSkipped, tookSecs));
// queries to demonstrate results of indexing
SolrQuery solrQuery = new SolrQuery("*:*");
solrQuery.setRows(0);
QueryResponse resp = solr.query(solrQuery);
SolrDocumentList hits = resp.getResults();
log.info("Match all docs distributed query found " + hits.getNumFound() + " docs.");
solrQuery.set("shards", "shard1");
resp = solr.query(solrQuery);
hits = resp.getResults();
log.info(
"Match all docs non-distributed query to shard1 found " + hits.getNumFound() + " docs.");
solrQuery.set("shards", "shard2");
resp = solr.query(solrQuery);
hits = resp.getResults();
log.info(
"Match all docs non-distributed query to shard2 found " + hits.getNumFound() + " docs.");
solr.shutdown();
}
public void merge(ResponseBuilder rb, ShardRequest sreq) {
// id to shard mapping, to eliminate any accidental dups
HashMap<Object, String> uniqueDoc = new HashMap<>();
NamedList<Object> shardInfo = null;
if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
shardInfo = new SimpleOrderedMap<>();
rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo);
}
IndexSchema schema = rb.req.getSchema();
SchemaField uniqueKeyField = schema.getUniqueKeyField();
long numFound = 0;
Float maxScore = null;
boolean partialResults = false;
List<ShardDoc> shardDocs = new ArrayList();
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs = null;
if (shardInfo != null) {
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
if (srsp.getException() != null) {
Throwable t = srsp.getException();
if (t instanceof SolrServerException) {
t = ((SolrServerException) t).getCause();
}
nl.add("error", t.toString());
StringWriter trace = new StringWriter();
t.printStackTrace(new PrintWriter(trace));
nl.add("trace", trace.toString());
if (srsp.getShardAddress() != null) {
nl.add("shardAddress", srsp.getShardAddress());
}
} else {
docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
nl.add("numFound", docs.getNumFound());
nl.add("maxScore", docs.getMaxScore());
nl.add("shardAddress", srsp.getShardAddress());
}
if (srsp.getSolrResponse() != null) {
nl.add("time", srsp.getSolrResponse().getElapsedTime());
}
shardInfo.add(srsp.getShard(), nl);
}
// now that we've added the shard info, let's only proceed if we have no error.
if (srsp.getException() != null) {
partialResults = true;
continue;
}
if (docs == null) { // could have been initialized in the shards info block above
docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
}
NamedList<?> responseHeader =
(NamedList<?>) srsp.getSolrResponse().getResponse().get("responseHeader");
if (responseHeader != null
&& Boolean.TRUE.equals(
responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
partialResults = true;
}
// calculate global maxScore and numDocsFound
if (docs.getMaxScore() != null) {
maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
}
numFound += docs.getNumFound();
SortSpec ss = rb.getSortSpec();
Sort sort = ss.getSort();
NamedList sortFieldValues =
(NamedList) (srsp.getSolrResponse().getResponse().get("merge_values"));
NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
List lst = (List) unmarshalledSortFieldValues.getVal(0);
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
String prevShard = uniqueDoc.put(id, srsp.getShard());
if (prevShard != null) {
// duplicate detected
numFound--;
// For now, just always use the first encountered since we can't currently
// remove the previous one added to the priority queue. If we switched
// to the Java5 PriorityQueue, this would be easier.
continue;
// make which duplicate is used deterministic based on shard
// if (prevShard.compareTo(srsp.shard) >= 0) {
// TODO: remove previous from priority queue
// continue;
// }
}
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.shard = srsp.getShard();
shardDoc.orderInShard = i;
Object scoreObj = lst.get(i);
if (scoreObj != null) {
shardDoc.score = ((Integer) scoreObj).floatValue();
}
shardDocs.add(shardDoc);
} // end for-each-doc-in-response
} // end for-each-response
Collections.sort(
shardDocs,
new Comparator<ShardDoc>() {
@Override
public int compare(ShardDoc o1, ShardDoc o2) {
if (o1.score < o2.score) {
return 1;
} else if (o1.score > o2.score) {
return -1;
} else {
return 0; // To change body of implemented methods use File | Settings | File
// Templates.
}
}
});
int resultSize = shardDocs.size();
Map<Object, ShardDoc> resultIds = new HashMap<>();
for (int i = 0; i < shardDocs.size(); i++) {
ShardDoc shardDoc = shardDocs.get(i);
shardDoc.positionInResponse = i;
// Need the toString() for correlation with other lists that must
// be strings (like keys in highlighting, explain, etc)
resultIds.put(shardDoc.id.toString(), shardDoc);
}
// Add hits for distributed requests
// https://issues.apache.org/jira/browse/SOLR-3518
rb.rsp.addToLog("hits", numFound);
SolrDocumentList responseDocs = new SolrDocumentList();
if (maxScore != null) responseDocs.setMaxScore(maxScore);
responseDocs.setNumFound(numFound);
responseDocs.setStart(0);
// size appropriately
for (int i = 0; i < resultSize; i++) responseDocs.add(null);
// save these results in a private area so we can access them
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb.setResponseDocs(responseDocs);
if (partialResults) {
rb.rsp
.getResponseHeader()
.add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
}
@Override
public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
QueryResponse response = null;
RecommendResponse searchResponse = new RecommendResponse();
long step0ElapsedTime = 0;
long step1ElapsedTime;
List<String> recommendations = new ArrayList<String>();
try {
// STEP 0 - get products from a user
if (query.getUser() != null) {
if (query.getProductIds() == null || query.getProductIds().size() == 0) {
if (alreadyBoughtProducts != null) {
query.setProductIds(alreadyBoughtProducts);
} else {
}
}
}
solrParams = getInteractionsFromMeAndUSersThatILikedOrCommented(query.getUser());
response =
SolrServiceContainer.getInstance()
.getSocialActionService()
.getSolrServer()
.query(solrParams);
step1ElapsedTime = response.getElapsedTime();
List<SocialAction> socialUsers = response.getBeans(SocialAction.class);
if (socialUsers.size() == 0) {
searchResponse.setNumFound(0);
searchResponse.setResultItems(recommendations);
searchResponse.setElapsedTime(-1);
return searchResponse;
}
SocialAction currentUserInteractions = socialUsers.get(0);
if (currentUserInteractions.getUserId().equals(query.getUser())) {
socialUsers.remove(0);
} else {
currentUserInteractions = null;
}
final Map<String, Integer> userInteractionMap = new HashMap<String, Integer>();
if (currentUserInteractions != null) {
List<String> usersThatPostedASnapshopToMe =
currentUserInteractions.getUsersThatPostedASnapshopToMe();
if (usersThatPostedASnapshopToMe != null) {
fillInteractions(usersThatPostedASnapshopToMe, userInteractionMap);
}
}
for (SocialAction socialUser : socialUsers) {
Integer userInteraction = userInteractionMap.get(socialUser.getUserId());
if (userInteraction == null) {
userInteraction = 0;
}
if (socialUser.getUsersThatPostedASnapshopToMe() != null) {
userInteraction +=
Collections.frequency(socialUser.getUsersThatPostedASnapshopToMe(), query.getUser());
}
userInteractionMap.put(socialUser.getUserId(), userInteraction);
}
Comparator<String> interactionCountComparator =
new Comparator<String>() {
@Override
public int compare(String a, String b) {
if (userInteractionMap.get(a) > userInteractionMap.get(b)) {
return -1;
} else if (userInteractionMap.get(a).equals(userInteractionMap.get(b))) {
return 0;
} else {
return 1;
}
}
};
TreeMap<String, Integer> sorted_map =
new TreeMap<String, Integer>(interactionCountComparator);
sorted_map.putAll(userInteractionMap);
solrParams = getSTEP2Params(query, maxReuslts, sorted_map);
// TODO Facet for confidence value
response =
SolrServiceContainer.getInstance().getResourceService().getSolrServer().query(solrParams);
// fill response object
List<Resource> beans = response.getBeans(Resource.class);
searchResponse.setResultItems(RecommendationQueryUtils.extractRecommendationIds(beans));
searchResponse.setElapsedTime(
step0ElapsedTime + step1ElapsedTime + response.getElapsedTime());
SolrDocumentList docResults = response.getResults();
searchResponse.setNumFound(docResults.getNumFound());
} catch (Exception e) {
System.out.println(solrParams);
e.printStackTrace();
searchResponse.setNumFound(0);
searchResponse.setResultItems(recommendations);
searchResponse.setElapsedTime(-1);
}
return searchResponse;
}
public static serverObjects respond(
@SuppressWarnings("unused") final RequestHeader header,
final serverObjects post,
final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final Segment segment = sb.index;
final SolrConnector connector = segment.fulltext().getDefaultConnector();
// avoid UNRESOLVED PATTERN
prop.put("url", "");
prop.put("citations", 0);
prop.put("sentences", 0);
DigestURL uri = null;
String url = "";
String hash = "";
int ch = 10;
boolean filter = false; // show cited sentences only
if (post != null) {
if (post.containsKey("url")) {
url = post.get("url");
if (!url.startsWith("http://")
&& !url.startsWith("https://")
&& !url.startsWith("ftp://")
&& !url.startsWith("smb://")
&& !url.startsWith("file://")) {
url = "http://" + url;
}
}
if (post.containsKey("hash")) {
hash = post.get("hash");
}
if (post.containsKey("ch")) {
ch = post.getInt("ch", ch);
}
filter = post.getBoolean("filter");
}
prop.put("filter", filter);
if (url.length() > 0) {
try {
uri = new DigestURL(url, null);
hash = ASCII.String(uri.hash());
} catch (final MalformedURLException e) {
}
}
if (uri == null && hash.length() > 0) {
try {
uri = sb.getURL(ASCII.getBytes(hash));
if (uri == null) {
connector.commit(true); // try again, that url can be fresh
uri = sb.getURL(ASCII.getBytes(hash));
}
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
if (uri == null) return prop; // no proper url addressed
url = uri.toNormalform(true);
prop.put("url", url);
// get the document from the index
SolrDocument doc;
try {
doc =
segment
.fulltext()
.getDefaultConnector()
.getDocumentById(
hash,
CollectionSchema.title.getSolrFieldName(),
CollectionSchema.text_t.getSolrFieldName());
} catch (final IOException e1) {
return prop;
}
@SuppressWarnings("unchecked")
ArrayList<String> title =
(ArrayList<String>) doc.getFieldValue(CollectionSchema.title.getSolrFieldName());
String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
ArrayList<String> sentences = new ArrayList<String>();
if (title != null) for (String s : title) if (s.length() > 0) sentences.add(s);
if (text != null && !text.isEmpty()) {
SentenceReader sr = new SentenceReader(text);
StringBuilder line;
while (sr.hasNext()) {
line = sr.next();
if (line.length() > 0) sentences.add(line.toString());
}
}
// for each line make a statistic about the number of occurrences somewhere else
OrderedScoreMap<String> scores =
new OrderedScoreMap<String>(null); // accumulates scores for citating urls
LinkedHashMap<String, Set<DigestURL>> sentenceOcc = new LinkedHashMap<String, Set<DigestURL>>();
for (String sentence : sentences) {
if (sentence == null || sentence.length() < 40) {
// do not count the very short sentences
sentenceOcc.put(sentence, null);
continue;
}
try {
sentence = sentence.replace('"', '\'');
SolrDocumentList doclist =
connector.getDocumentListByQuery(
"text_t:\"" + sentence + "\"",
CollectionSchema.url_chars_i.getSolrFieldName() + " asc",
0,
100,
CollectionSchema.sku.getSolrFieldName());
int count = (int) doclist.getNumFound();
if (count > 0) {
Set<DigestURL> list = new TreeSet<DigestURL>();
for (SolrDocument d : doclist) {
String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName());
if (u == null || u.equals(url)) continue;
scores.inc(u);
try {
list.add(new DigestURL(u, null));
} catch (final MalformedURLException e) {
}
}
sentenceOcc.put(sentence, list);
}
} catch (final Throwable ee) {
}
}
sentences.clear(); // we do not need this again
// iterate the sentences
int i = 0;
int sentenceNr = 0;
for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) {
Set<DigestURL> app = se.getValue();
if (filter) { // prepare list, only include sentence with citation
if (app != null && app.size() > 0) {
StringBuilder dd = new StringBuilder(se.getKey());
prop.put("sentences_" + i + "_dt", sentenceNr);
dd.append("<br/>appears in:");
for (DigestURL u : app) {
if (u != null) {
dd.append(" <a href=\"")
.append(u.toNormalform(false))
.append("\">")
.append(u.getHost())
.append("</a>");
}
}
prop.put("sentences_" + i + "_dd", dd.toString());
i++;
}
} else { // prepare list, include all sentences
StringBuilder dd = new StringBuilder(se.getKey());
prop.put("sentences_" + i + "_dt", sentenceNr);
if (app != null && app.size() > 0) {
dd.append("<br/>appears in:");
for (DigestURL u : app) {
if (u != null) {
dd.append(" <a href=\"")
.append(u.toNormalform(false))
.append("\">")
.append(u.getHost())
.append("</a>");
}
}
}
prop.put("sentences_" + i + "_dd", dd.toString());
i++;
}
sentenceNr++;
}
prop.put("sentences", i);
// iterate the citations in order of number of citations
i = 0;
for (String u : scores.keyList(false)) {
try {
DigestURL uu = new DigestURL(u, null);
prop.put("citations_" + i + "_dt", "<a href=\"" + u + "\">" + u + "</a>");
StringBuilder dd = new StringBuilder();
dd.append("makes ")
.append(Integer.toString(scores.get(u)))
.append(" citations: of ")
.append(url);
for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) {
Set<DigestURL> occurls = se.getValue();
if (occurls != null && occurls.contains(uu))
dd.append("<br/><a href=\"/solr/select?q=text_t:%22")
.append(se.getKey().replace('"', '\''))
.append("%22&rows=100&grep=&wt=grephtml\">")
.append(se.getKey())
.append("</a>");
}
prop.put("citations_" + i + "_dd", dd.toString());
i++;
} catch (final MalformedURLException e) {
}
}
prop.put("citations", i);
// find similar documents from different hosts
i = 0;
for (String u : scores.keyList(false)) {
if (scores.get(u) < ch) continue;
try {
DigestURL uu = new DigestURL(u, null);
if (uu.getOrganization().equals(uri.getOrganization())) continue;
prop.put("similar_links_" + i + "_url", u);
i++;
} catch (final MalformedURLException e) {
}
}
prop.put("similar_links", i);
prop.put("similar", i > 0 ? 1 : 0);
// return rewrite properties
return prop;
}
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
SortSpec ss = rb.getSortSpec();
Sort sort = ss.getSort();
SortField[] sortFields = null;
if (sort != null) sortFields = sort.getSort();
else {
sortFields = new SortField[] {SortField.FIELD_SCORE};
}
SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
// id to shard mapping, to eliminate any accidental dups
HashMap<Object, String> uniqueDoc = new HashMap<Object, String>();
// Merge the docs via a priority queue so we don't have to sort *all* of the
// documents... we only need to order the top (rows+start)
ShardFieldSortedHitQueue queue;
queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
long numFound = 0;
Float maxScore = null;
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs =
(SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
// calculate global maxScore and numDocsFound
if (docs.getMaxScore() != null) {
maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
}
numFound += docs.getNumFound();
NamedList sortFieldValues =
(NamedList) (srsp.getSolrResponse().getResponse().get("sort_values"));
// go through every doc in this response, construct a ShardDoc, and
// put it in the priority queue so it can be ordered.
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
String prevShard = uniqueDoc.put(id, srsp.getShard());
if (prevShard != null) {
// duplicate detected
numFound--;
// For now, just always use the first encountered since we can't currently
// remove the previous one added to the priority queue. If we switched
// to the Java5 PriorityQueue, this would be easier.
continue;
// make which duplicate is used deterministic based on shard
// if (prevShard.compareTo(srsp.shard) >= 0) {
// TODO: remove previous from priority queue
// continue;
// }
}
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.shard = srsp.getShard();
shardDoc.orderInShard = i;
Object scoreObj = doc.getFieldValue("score");
if (scoreObj != null) {
if (scoreObj instanceof String) {
shardDoc.score = Float.parseFloat((String) scoreObj);
} else {
shardDoc.score = (Float) scoreObj;
}
}
shardDoc.sortFieldValues = sortFieldValues;
queue.insertWithOverflow(shardDoc);
} // end for-each-doc-in-response
} // end for-each-response
// The queue now has 0 -> queuesize docs, where queuesize <= start + rows
// So we want to pop the last documents off the queue to get
// the docs offset -> queuesize
int resultSize = queue.size() - ss.getOffset();
resultSize = Math.max(0, resultSize); // there may not be any docs in range
Map<Object, ShardDoc> resultIds = new HashMap<Object, ShardDoc>();
for (int i = resultSize - 1; i >= 0; i--) {
ShardDoc shardDoc = (ShardDoc) queue.pop();
shardDoc.positionInResponse = i;
// Need the toString() for correlation with other lists that must
// be strings (like keys in highlighting, explain, etc)
resultIds.put(shardDoc.id.toString(), shardDoc);
}
SolrDocumentList responseDocs = new SolrDocumentList();
if (maxScore != null) responseDocs.setMaxScore(maxScore);
responseDocs.setNumFound(numFound);
responseDocs.setStart(ss.getOffset());
// size appropriately
for (int i = 0; i < resultSize; i++) responseDocs.add(null);
// save these results in a private area so we can access them
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb._responseDocs = responseDocs;
}
@RequestMapping(value = "/searchtweetresponse", method = RequestMethod.GET)
@ResponseBody
public String getTweets(
@RequestParam("search") String searchKey,
@RequestParam("start") int start,
@RequestParam("end") int end) {
logger.info("Welcome to Tweets");
JSONObject finalObject = new JSONObject();
int startRow = 0;
int endRow = 10;
// String tweetType="*";
if (start > startRow) {
startRow = start;
}
if (end > endRow) {
endRow = end;
}
SolrQuery query = new SolrQuery("tweet_content:*" + searchKey + "*");
query.setStart(startRow);
query.setRows(endRow);
HomeBean bean = new HomeBean();
Collection<JSONObject> objectList = new ArrayList<JSONObject>();
try {
QueryResponse response = homeservice.getServiceResponse(query);
if (response != null) {
SolrDocumentList responseList = response.getResults();
long totalCount = responseList.getNumFound();
if (totalCount > 0) {
bean.setResultCount(totalCount);
}
finalObject.put("total", totalCount);
for (SolrDocument document : responseList) {
JSONObject json = new JSONObject();
json.put("id", document.getFieldValue("id"));
json.put("tweet", document.getFieldValue("tweet_content"));
objectList.add(json);
}
}
} catch (Exception e) {
e.printStackTrace();
}
finalObject.put("tweetList", objectList);
query.set("facet", true);
query.addFacetField("tweetPostedTime");
query.addSort("tweetPostedTime", SolrQuery.ORDER.asc);
query.setRows(0);
QueryResponse response = homeservice.getServiceResponse(query);
// 2015-07-30T12:10:31Z
DateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd");
SimpleDateFormat newFormat = new SimpleDateFormat("MM-dd-yyyy");
DateFormat dateFormat2 = new SimpleDateFormat("E MMM d hh:mm:ss zzz yyyy");
List<Date> myList = new ArrayList<Date>();
// Collection<JSONObject> tempList = new ArrayList<JSONObject>();
Collection<JSONObject> dateCountList = new ArrayList<JSONObject>();
JSONObject tempObj = new JSONObject();
try {
if (response != null) {
FacetField timeFacet = response.getFacetField("tweetPostedTime");
List<Count> list = timeFacet.getValues();
Iterator itr = list.iterator();
// Collection<String> productKeys = new ArrayList<String>();
// Collection<String> smartPhoneKeys = new ArrayList<String>();
while (itr.hasNext()) {
Count timeObj = (Count) itr.next();
if (timeObj != null) {
String timeName = timeObj.getName();
String originalTime = timeName.split("T")[0];
Date beforeParse = dateFormat1.parse(originalTime);
myList.add(beforeParse);
JSONObject timeObject = new JSONObject();
timeObject.put("date", newFormat.format(beforeParse));
timeObject.put("count", timeObj.getCount());
// tempList.add(timeObject);
tempObj.put(newFormat.format(beforeParse), timeObject);
}
}
}
Collections.sort(myList);
Iterator<Date> itr = myList.iterator();
while (itr.hasNext()) {
Date inputDate = dateFormat2.parse(itr.next().toString());
String tempKey = newFormat.format(inputDate);
System.out.println(tempKey);
JSONObject newObj = (JSONObject) tempObj.get(tempKey);
dateCountList.add(newObj);
}
finalObject.put("dateList", dateCountList);
} catch (Exception e) {
logger.error(e.getMessage());
e.printStackTrace();
}
logger.info("bye Tweets");
return finalObject.toString();
}
