public JSONArray Search(String query, String sort) { SolrServer server; SolrDocumentList docs = null; String Date_format = null; // for Date格式轉換 JSONArray jsonArray = new JSONArray(); try { server = new HttpSolrServer(url); // add SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(query); solrQuery.setStart(_start); solrQuery.setRows(_nbDocuments); solrQuery.setRequestHandler("query"); solrQuery.set("fl", "*,score"); // 設定fl參數,指定要傳回哪個field的資料,這裡設定所有field與score if (_fq1 != "") { solrQuery.addFilterQuery("ProductName:" + _fq1); } if (_fq2 != "") { solrQuery.addFilterQuery("publishedDate:" + _fq2); } if (sort != null) { solrQuery.addSortField(sort, ORDER.asc); } solrQuery.setRequestHandler("/browse"); response = server.query(solrQuery); docs = response.getResults(); if (docs != null) { System.out.println( docs.getNumFound() + " documents found, " + docs.size() + " returned : "); setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端 for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); JSONObject jsonObject = new JSONObject(); for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) { Entry<String, Object> entry = it2.next(); if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串,方便前端呈現 Date_format = entry.getValue().toString(); jsonObject.put(entry.getKey(), Date_format); } else { // 一般情況 jsonObject.put(entry.getKey(), entry.getValue()); } } System.out.print("\n"); // 將總共找到幾筆資料存在jsonarray的最後面傳給前端 jsonObject.put("TotalResultFound", docs.getNumFound()); jsonArray.add(jsonObject); } } } catch (SolrServerException e) { // TODO Auto-generated catch block e.printStackTrace(); } return jsonArray; }
@Test(dependsOnMethods = "testAddDocument") public void testBasicQuery() throws SolrServerException { SolrServer server = solrServerFactory.getServer(); SolrParams params = new SolrQuery("*:*"); SolrDocumentList results = server.query(params).getResults(); assertTrue(results.getNumFound() > 0, "didn't return any results"); }
protected List<Vector> processQueryResponse(QueryResponse resp) { NamedList<Object> response = resp.getResponse(); NamedList<Object> termVectorsNL = (NamedList<Object>) response.get("termVectors"); if (termVectorsNL == null) throw new RuntimeException( "No termVectors in response! " + "Please check your query to make sure it is requesting term vector information from Solr correctly."); List<Vector> termVectors = new ArrayList<Vector>(termVectorsNL.size()); Iterator<Map.Entry<String, Object>> iter = termVectorsNL.iterator(); while (iter.hasNext()) { Map.Entry<String, Object> next = iter.next(); String nextKey = next.getKey(); Object nextValue = next.getValue(); if (nextValue instanceof NamedList) { NamedList nextList = (NamedList) nextValue; Object fieldTerms = nextList.get(field); if (fieldTerms != null && fieldTerms instanceof NamedList) { termVectors.add( SolrTermVector.newInstance(nextKey, hashingTF, (NamedList<Object>) fieldTerms)); } } } SolrDocumentList docs = resp.getResults(); totalDocs = docs.getNumFound(); return termVectors; }
public SearchResponse search(SearchRequest request) { SolrQuery query = new SolrQuery(); query.setRows(request.getLimit()); query.setStart(request.getOffset() * request.getLimit()); BooleanQuery aggregate = new BooleanQuery(); if (!StringUtils.isBlank(request.getText())) { TermQuery termQuery = new TermQuery(new Term("", request.getText())); aggregate.add(termQuery, BooleanClause.Occur.SHOULD); } if (!StringUtils.isBlank(request.getEventId())) { aggregate.add( new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST); } query.setQuery(aggregate.toString()); log.info("QUERY IS: " + query.toString()); try { QueryResponse queryResponse = solrServer.query(query); log.info("RESPONSE FROM QUERY WAS: " + queryResponse); SolrDocumentList results = queryResponse.getResults(); log.info("RESULTS WAS: " + results); ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>(); for (SolrDocument result : results) { hits.add( new SearchResponse.Hit( ObjectType.session, String.valueOf(result.getFieldValue("id")), null)); } return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits); } catch (SolrServerException e) { e.printStackTrace(); } return null; }
/** * Returns SearchResultsSlim for a given SolrDocumentList. A "slimmer" results object without the * "items" wrapper element is created for better transform to json. * * @param doc solr document list to build results * @return the SearchResultsSlim object for this solr result * @see SearchResultsSlim */ private SearchResultsSlim buildSlimResults(SolrDocumentList docs) { SearchResultsSlim results = new SearchResultsSlim(); Pagination pagination = new Pagination(); pagination.setNumFound(docs.getNumFound()); pagination.setStart(docs.getStart()); pagination.setRows(limit); // List<ModsType> modsTypes = new ArrayList<ModsType>(); List<Item> items = new ArrayList<Item>(); for (final SolrDocument doc : docs) { Item item = new Item(); ModsType modsType = null; try { modsType = (new ItemDAO()).getModsType(doc); } catch (JAXBException je) { log.error(je.getMessage()); je.printStackTrace(); } item.setModsType(modsType); items.add(item); } results.setItems(items); results.setPagination(pagination); if (facet != null) results.setFacet(facet); return results; }
@Test(dependsOnMethods = "testBasicQuery", dataProvider = "documentMap") public void testIdQuery(Map<String, String[]> document) throws SolrServerException { String id = document.get("id")[0]; SolrServer server = solrServerFactory.getServer(); SolrParams params = new SolrQuery("id:" + id); SolrDocumentList results = server.query(params).getResults(); assertEquals(results.getNumFound(), 1, "didn't find article by id"); }
public void writeSolrDocumentList(SolrDocumentList docs) throws IOException { writeTag(SOLRDOCLST); List<Number> l = new ArrayList<>(3); l.add(docs.getNumFound()); l.add(docs.getStart()); l.add(docs.getMaxScore()); writeArray(l); writeArray(docs); }
@Override public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) { productsToFilter = new ArrayList<String>(); String queryString = "*:*"; String sortCriteria = "user_count_purchased desc"; ModifiableSolrParams solrParams = new ModifiableSolrParams(); QueryResponse response = null; RecommendResponse searchResponse = new RecommendResponse(); try { String filterQueryString = RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter); if (query.getProductIds() != null && query.getProductIds().size() > 0) { productsToFilter.addAll(query.getProductIds()); } if (query.getUser() != null) { if (alreadyBoughtProducts != null) { productsToFilter.addAll(alreadyBoughtProducts); } } if (productsToFilter != null && productsToFilter.size() > 0) { if (filterQueryString.length() > 0) { filterQueryString += " OR "; } filterQueryString += RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter); } solrParams.set("q", queryString); solrParams.set("fq", filterQueryString); solrParams.set("sort", sortCriteria); solrParams.set("rows", maxReuslts); response = SolrServiceContainer.getInstance() .getRecommendService() .getSolrServer() .query(solrParams); // fill response object List<String> extractedRecommendations = RecommendationQueryUtils.extractRecommendationIds( response.getBeans(CustomerAction.class)); searchResponse.setResultItems(extractedRecommendations); searchResponse.setElapsedTime(response.getElapsedTime()); SolrDocumentList docResults = response.getResults(); searchResponse.setNumFound(docResults.getNumFound()); } catch (SolrServerException e) { e.printStackTrace(); searchResponse.setNumFound(0); searchResponse.setResultItems(new ArrayList<String>()); searchResponse.setElapsedTime(-1); } return searchResponse; }
@Override public void writeSolrDocumentList( String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException { boolean includeScore = false; if (fields != null) { includeScore = fields.contains("score"); if (fields.size() == 0 || (fields.size() == 1 && includeScore) || fields.contains("*")) { fields = null; // null means return all stored fields } } int sz = docs.size(); writeMapOpener(includeScore ? 4 : 3); incLevel(); writeKey("numFound", false); writeLong(null, docs.getNumFound()); writeMapSeparator(); writeKey("start", false); writeLong(null, docs.getStart()); if (includeScore && docs.getMaxScore() != null) { writeMapSeparator(); writeKey("maxScore", false); writeFloat(null, docs.getMaxScore()); } writeMapSeparator(); // indent(); writeKey("docs", false); writeArrayOpener(sz); incLevel(); boolean first = true; SolrIndexSearcher searcher = req.getSearcher(); for (SolrDocument doc : docs) { if (first) { first = false; } else { writeArraySeparator(); } indent(); writeSolrDocument(null, doc, fields, otherFields); } decLevel(); writeArrayCloser(); if (otherFields != null) { writeMap(null, otherFields, true, false); } decLevel(); indent(); writeMapCloser(); }
@Override public <T> ScoredPage<T> queryForPage(Query query, Class<T> clazz) { Assert.notNull(query, "Query must not be 'null'."); Assert.notNull(clazz, "Target class must not be 'null'."); QueryResponse response = query(query); List<T> beans = convertQueryResponseToBeans(response, clazz); SolrDocumentList results = response.getResults(); return new SolrResultPage<T>( beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore()); }
@Test public void testGetIdsWithParams() throws Exception { SolrDocumentList rsp = getSolrClient().getById(Arrays.asList("0", "1", "2"), params(CommonParams.FL, "id")); assertEquals(2, rsp.getNumFound()); assertEquals("1", rsp.get(0).get("id")); assertNull("This field should have been removed from the response.", rsp.get(0).get("term_s")); assertNull("This field should have been removed from the response.", rsp.get(0).get("term2_s")); assertEquals("2", rsp.get(1).get("id")); assertNull("This field should have been removed from the response.", rsp.get(1).get("term_s")); assertNull("This field should have been removed from the response.", rsp.get(1).get("term2_s")); }
/** SRN-96 */ @Test public void testASCIIFoldingExpansion() throws IOException, SolrServerException { this.addJsonString("1", " { \"value\" : \"cafe\" } "); this.addJsonString("2", " { \"value\" : \"café\" } "); SolrQuery query = new SolrQuery(); query.setQuery("cafe"); query.setRequestHandler("keyword"); query.setIncludeScore(true); // should match the two documents, with same score QueryResponse response = getWrapper().getServer().query(query); SolrDocumentList docList = response.getResults(); assertEquals(2, docList.getNumFound()); float score1 = (Float) docList.get(0).getFieldValue("score"); float score2 = (Float) docList.get(1).getFieldValue("score"); Assert.assertTrue("Score should be identical", score1 == score2); // should match the two documents, but should assign different score // id2 should receive better score than id1 query = new SolrQuery(); query.setQuery("café"); query.setRequestHandler("keyword"); query.setIncludeScore(true); response = getWrapper().getServer().query(query); docList = response.getResults(); assertEquals(2, docList.getNumFound()); if (docList.get(0).getFieldValue("url").equals("id1")) { score1 = (Float) docList.get(0).getFieldValue("score"); score2 = (Float) docList.get(1).getFieldValue("score"); } else { score2 = (Float) docList.get(0).getFieldValue("score"); score1 = (Float) docList.get(1).getFieldValue("score"); } Assert.assertTrue("id2 should get higher score than id1", score1 < score2); }
// 查询 @Test public void testSolrForSearch() throws Exception { SolrQuery params = new SolrQuery(); params.set("q", "*:*"); QueryResponse response = solrServer.query(params); // 结果集 SolrDocumentList docs = response.getResults(); long count = docs.getNumFound(); System.out.println("结果集有 :" + count); for (SolrDocument doc : docs) { String id = (String) doc.get("id"); String name = (String) doc.get("name"); System.out.println("id :" + id + "---" + "name :" + name); } }
@Override public <T> HighlightPage<T> queryForHighlightPage(HighlightQuery query, Class<T> clazz) { Assert.notNull(query, "Query must not be 'null'."); Assert.notNull(clazz, "Target class must not be 'null'."); QueryResponse response = query(query); List<T> beans = convertQueryResponseToBeans(response, clazz); SolrDocumentList results = response.getResults(); SolrResultPage<T> page = new SolrResultPage<T>( beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore()); ResultHelper.convertAndAddHighlightQueryResponseToResultPage(response, page); return page; }
@Test public void testGetIds() throws Exception { SolrDocumentList rsp = getSolrClient().getById(Arrays.asList("0", "1", "2", "3", "4")); assertEquals(3, rsp.getNumFound()); assertEquals("1", rsp.get(0).get("id")); assertEquals("Microsoft", rsp.get(0).get("term_s")); assertEquals("MSFT", rsp.get(0).get("term2_s")); assertEquals("2", rsp.get(1).get("id")); assertEquals("Apple", rsp.get(1).get("term_s")); assertEquals("AAPL", rsp.get(1).get("term2_s")); assertEquals("3", rsp.get(2).get("id")); assertEquals("Yahoo", rsp.get(2).get("term_s")); assertEquals("YHOO", rsp.get(2).get("term2_s")); }
@Override public <T> FacetPage<T> queryForFacetPage(FacetQuery query, Class<T> clazz) { Assert.notNull(query, "Query must not be 'null'."); Assert.notNull(clazz, "Target class must not be 'null'."); QueryResponse response = query(query); List<T> beans = convertQueryResponseToBeans(response, clazz); SolrDocumentList results = response.getResults(); SolrResultPage<T> page = new SolrResultPage<T>( beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore()); page.addAllFacetFieldResultPages( ResultHelper.convertFacetQueryResponseToFacetPageMap(query, response)); page.addAllFacetPivotFieldResult( ResultHelper.convertFacetQueryResponseToFacetPivotMap(query, response)); page.setFacetQueryResultPage( ResultHelper.convertFacetQueryResponseToFacetQueryResult(query, response)); return page; }
private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest) throws SolrServerException, IOException { // verify the join with fromIndex works final String fromQ = "match_s:c^2"; CloudSolrClient client = cluster.getSolrClient(); { final String joinQ = "{!join " + anyScoreMode(isScoresTest) + "from=join_s fromIndex=" + fromColl + " to=join_s}" + fromQ; QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); QueryResponse rsp = new QueryResponse(client.request(qr), client); SolrDocumentList hits = rsp.getResults(); assertTrue("Expected 1 doc, got " + hits, hits.getNumFound() == 1); SolrDocument doc = hits.get(0); assertEquals(toDocId, doc.getFirstValue("id")); assertEquals("b", doc.getFirstValue("get_s")); assertScore(isScoresTest, doc); } // negative test before creating an alias checkAbsentFromIndex(fromColl, toColl, isScoresTest); // create an alias for the fromIndex and then query through the alias String alias = fromColl + "Alias"; CollectionAdminRequest.CreateAlias request = CollectionAdminRequest.createAlias(alias, fromColl); request.process(client); { final String joinQ = "{!join " + anyScoreMode(isScoresTest) + "from=join_s fromIndex=" + alias + " to=join_s}" + fromQ; final QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); final QueryResponse rsp = new QueryResponse(client.request(qr), client); final SolrDocumentList hits = rsp.getResults(); assertTrue("Expected 1 doc", hits.getNumFound() == 1); SolrDocument doc = hits.get(0); assertEquals(toDocId, doc.getFirstValue("id")); assertEquals("b", doc.getFirstValue("get_s")); assertScore(isScoresTest, doc); } // negative test after creating an alias checkAbsentFromIndex(fromColl, toColl, isScoresTest); { // verify join doesn't work if no match in the "from" index final String joinQ = "{!join " + (anyScoreMode(isScoresTest)) + "from=join_s fromIndex=" + fromColl + " to=join_s}match_s:d"; final QueryRequest qr = new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score")); final QueryResponse rsp = new QueryResponse(client.request(qr), client); final SolrDocumentList hits = rsp.getResults(); assertTrue("Expected no hits", hits.getNumFound() == 0); } }
/** * Sends an email to the given e-person with details of new items in the given dspace object (MUST * be a community or a collection), items that appeared yesterday. No e-mail is sent if there * aren't any new items in any of the dspace objects. * * @param context DSpace context object * @param eperson eperson to send to * @param rpkeys List of DSpace Objects * @param test * @throws SearchServiceException */ public static void sendEmail( Researcher researcher, Context context, EPerson eperson, List<String> rpkeys, boolean test, List<String> relationFields) throws IOException, MessagingException, SQLException, SearchServiceException { CrisSearchService searchService = researcher.getCrisSearchService(); // Get a resource bundle according to the eperson language preferences Locale supportedLocale = I18nUtil.getEPersonLocale(eperson); StringBuffer emailText = new StringBuffer(); boolean isFirst = true; for (String rpkey : rpkeys) { SolrQuery query = new SolrQuery(); query.setFields("search.resourceid"); query.addFilterQuery( "{!field f=search.resourcetype}" + Constants.ITEM, "{!field f=inarchive}true"); for (String tmpRelations : relationFields) { String fq = "{!field f=" + tmpRelations + "}" + rpkey; query.addFilterQuery(fq); } query.setRows(Integer.MAX_VALUE); if (ConfigurationManager.getBooleanProperty("eperson.subscription.onlynew", false)) { // get only the items archived yesterday query.setQuery("dateaccessioned:(NOW/DAY-1DAY)"); } else { // get all item modified yesterday but not published the day // before // and all the item modified today and archived yesterday query.setQuery( "(item.lastmodified:(NOW/DAY-1DAY) AND dateaccessioned:(NOW/DAY-1DAY)) OR ((item.lastmodified:(NOW/DAY) AND dateaccessioned:(NOW/DAY-1DAY)))"); } QueryResponse qResponse = searchService.search(query); SolrDocumentList results = qResponse.getResults(); // Only add to buffer if there are new items if (results.getNumFound() > 0) { if (!isFirst) { emailText.append("\n---------------------------------------\n"); } else { isFirst = false; } emailText .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.new-items", supportedLocale)) .append(" ") .append(rpkey) .append(": ") .append(results.getNumFound()) .append("\n\n"); for (SolrDocument solrDoc : results) { Item item = Item.find(context, (Integer) solrDoc.getFieldValue("search.resourceid")); DCValue[] titles = item.getDC("title", null, Item.ANY); emailText .append(" ") .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.title", supportedLocale)) .append(" "); if (titles.length > 0) { emailText.append(titles[0].value); } else { emailText.append( I18nUtil.getMessage("org.dspace.eperson.Subscribe.untitled", supportedLocale)); } DCValue[] authors = item.getDC("contributor", Item.ANY, Item.ANY); if (authors.length > 0) { emailText .append("\n ") .append( I18nUtil.getMessage("org.dspace.eperson.Subscribe.authors", supportedLocale)) .append(" ") .append(authors[0].value); for (int k = 1; k < authors.length; k++) { emailText.append("\n ").append(authors[k].value); } } emailText .append("\n ") .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.id", supportedLocale)) .append(" ") .append(HandleManager.getCanonicalForm(item.getHandle())) .append("\n\n"); context.removeCached(item, item.getID()); } } } // Send an e-mail if there were any new items if (emailText.length() > 0) { if (test) { log.info(LogManager.getHeader(context, "subscription:", "eperson=" + eperson.getEmail())); log.info(LogManager.getHeader(context, "subscription:", "text=" + emailText.toString())); } else { Email email = ConfigurationManager.getEmail( I18nUtil.getEmailFilename(supportedLocale, "subscription")); email.addRecipient(eperson.getEmail()); email.addArgument(emailText.toString()); email.send(); log.info( LogManager.getHeader(context, "sent_subscription", "eperson_id=" + eperson.getID())); } } }
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { response.setContentType(XML_RESPONSE_HEADER); // Talkback happens in XML form. response.setCharacterEncoding("UTF-8"); // Unicode++ request.setCharacterEncoding("UTF-8"); PrintWriter out = null; // The talkback buffer. // handle startrecord Integer startRecord = 0; if (!(request.getParameter("startRecord") == null)) { try { startRecord = Integer.parseInt(request.getParameter("startRecord")) - 1; } catch (NumberFormatException e) { startRecord = 0; } } // maximumrecords Integer maximumRecords = Integer.parseInt(this.config.getProperty("default_maximumRecords")); if (!(request.getParameter("maximumRecords") == null)) { maximumRecords = Integer.parseInt(request.getParameter("maximumRecords")); } // operation String operation = request.getParameter("operation"); // x_collection String x_collection = request.getParameter("x-collection"); if (x_collection == null) x_collection = this.config.getProperty("default_x_collection"); if (x_collection == null) operation = null; // sortkeys String sortKeys = request.getParameter("sortKeys"); // sortorder String sortOrder = request.getParameter("sortOrder"); // recordschema String recordSchema = request.getParameter("recordSchema"); if (recordSchema == null) recordSchema = "dc"; if (recordSchema.equalsIgnoreCase("dcx")) { recordSchema = "dcx"; } if (recordSchema.equalsIgnoreCase("solr")) { recordSchema = "solr"; } // query request String query = request.getParameter("query"); String q = request.getParameter("q"); // who is requestor ? String remote_ip = request.getHeader("X-FORWARDED-FOR"); if (remote_ip == null) { remote_ip = request.getRemoteAddr().trim(); } else { remote_ip = request.getHeader("X-FORWARDED-FOR"); } // handle debug Boolean debug = Boolean.parseBoolean(request.getParameter("debug")); if (!debug) { out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true); } // handle query if ((query == null) && (q != null)) { query = q; } else { if ((query != null) && (q == null)) { q = query; } else { operation = null; } } // handle operation if (operation == null) { if (query != null) { operation = "searchRetrieve"; } else { operation = "explain"; } } // searchRetrieve if (operation.equalsIgnoreCase("searchRetrieve")) { if (query == null) { operation = "explain"; log.debug(operation + ":" + query); } } // start talking back. String[] sq = {""}; String solrquery = ""; // facet String facet = null; List<FacetField> fct = null; if (request.getParameter("facet") != null) { facet = request.getParameter("facet"); log.debug("facet : " + facet); } if (operation == null) { operation = "searchretrieve"; } else { // explain response if (operation.equalsIgnoreCase("explain")) { log.debug("operation = explain"); out.write("<srw:explainResponse xmlns:srw=\"http://www.loc.gov/zing/srw/\">"); out.write("</srw:explainResponse>"); } else { // DEBUG routine operation = "searchretrieve"; String triplequery = null; if (query.matches(".*?\\[.+?\\].*?")) { // New symantic syntax triplequery = symantic_query(query); query = query.split("\\[")[0] + " " + triplequery; log.fatal(triplequery); solrquery = CQLtoLucene.translate(query, log, config); } else { solrquery = CQLtoLucene.translate(query, log, config); } log.debug(solrquery); if (debug == true) { response.setContentType(HTML_RESPONSE_HEADER); out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true); out.write("<html><body>\n\n"); out.write("'" + remote_ip + "'<br>\n"); out.write("<form action='http://www.kbresearch.nl/kbSRU'>"); out.write("<input type=text name=q value='" + query + "' size=120>"); out.write("<input type=hidden name=debug value=True>"); out.write("<input type=submit>"); out.write("<table border=1><tr><td>"); out.write("q</td><td>" + query + "</td></tr><tr>"); out.write("<td>query out</td><td>" + URLDecoder.decode(solrquery) + "</td></tr>"); out.write( "<tr><td>SOLR_URL</td><td> <a href='" + this.config.getProperty( "collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/?q=" + solrquery + "'>" + this.config.getProperty( "collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/select/?q=" + solrquery + "</a><br>" + this.config.getProperty("solr_url") + solrquery + "</td></tr>"); out.write( "<b>SOLR_QUERY</b> : <BR> <iframe width=900 height=400 src='" + this.config.getProperty( "collection." + x_collection.toLowerCase() + ".solr_baseurl") + "/../?q=" + solrquery + "'></iframe><BR>"); out.write( "<b>SRU_QUERY</b> : <BR> <a href=" + this.config.getProperty("baseurl") + "?q=" + query + "'>" + this.config.getProperty("baseurl") + "?q=" + query + "</a><br><iframe width=901 height=400 src='http://www.kbresearch.nl/kbSRU/?q=" + query + "'></iframe><BR>"); out.write( "<br><b>JSRU_QUERY</b> : <BR><a href='http://jsru.kb.nl/sru/?query=" + query + "&x-collection=" + x_collection + "'>http://jsru.kb.nl/sru/?query=" + query + "&x-collection=GGC</a><br><iframe width=900 height=400 src='http://jsru.kb.nl/sru/?query=" + query + "&x-collection=GGC'></iframe>"); } else { // XML SearchRetrieve response String url = this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl"); String buffer = ""; CommonsHttpSolrServer server = null; server = new CommonsHttpSolrServer(url); log.fatal("URSING " + url); server.setParser(new XMLResponseParser()); int numfound = 0; try { SolrQuery do_query = new SolrQuery(); do_query.setQuery(solrquery); do_query.setRows(maximumRecords); do_query.setStart(startRecord); if ((sortKeys != null) && (sortKeys.length() > 1)) { if (sortOrder != null) { if (sortOrder.equals("asc")) { do_query.setSortField(sortKeys, SolrQuery.ORDER.asc); } if (sortOrder.equals("desc")) { do_query.setSortField(sortKeys, SolrQuery.ORDER.desc); } } else { for (String str : sortKeys.trim().split(",")) { str = str.trim(); if (str.length() > 1) { if (str.equals("date")) { do_query.setSortField("date_date", SolrQuery.ORDER.desc); log.debug("SORTORDERDEBUG | DATE! " + str + " | "); break; } else { do_query.setSortField(str + "_str", SolrQuery.ORDER.asc); log.debug("SORTORDERDEBUG | " + str + " | "); break; } } } } } if (facet != null) { if (facet.indexOf(",") > 1) { for (String str : facet.split(",")) { if (str.indexOf("date") > 1) { do_query.addFacetField(str); } else { do_query.addFacetField(str); } // do_query.setParam("facet.method", "enum"); } // q.setFacetSort(false); } else { do_query.addFacetField(facet); } do_query.setFacet(true); do_query.setFacetMinCount(1); do_query.setFacetLimit(-1); } log.fatal(solrquery); QueryResponse rsp = null; boolean do_err = false; boolean do_sugg = false; SolrDocumentList sdl = null; String diag = ""; StringBuffer suggest = new StringBuffer(""); String content = "1"; SolrQuery spellq = do_query; try { rsp = server.query(do_query); } catch (SolrServerException e) { String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", "0"); out.write(header); diag = this.SRW_DIAG.replaceAll("\\$error", e.getMessage()); do_err = true; rsp = null; } log.fatal("query done.."); if (!(do_err)) { // XML dc response SolrDocumentList docs = rsp.getResults(); numfound = (int) docs.getNumFound(); int count = startRecord; String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", Integer.toString(numfound)); out.write(header); out.write("<srw:records>"); Iterator<SolrDocument> iter = rsp.getResults().iterator(); while (iter.hasNext()) { count += 1; if (recordSchema.equalsIgnoreCase("dc")) { SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" xmlns:facets=\"info:srw/extension/4/facets\" >"); StringBuffer result = new StringBuffer(""); construct_lucene_dc(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write( "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } if (recordSchema.equalsIgnoreCase("solr")) { SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/solr</srw:recordSchema>"); out.write("<srw:recordData xmlns:expand=\"http://www.kbresearch.nl/expand\">"); StringBuffer result = new StringBuffer(""); construct_lucene_solr(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write( "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } if (recordSchema.equalsIgnoreCase("dcx")) { // XML dcx response out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData><srw_dc:dc xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/marc.relators/\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); SolrDocument resultDoc = iter.next(); content = (String) resultDoc.getFieldValue("id"); String dcx_data = helpers.getOAIdcx( "http://services.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content, log); if (x_collection.equalsIgnoreCase("ggc-thes")) { dcx_data = helpers.getOAIdcx( "http://serviceso.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content, log); } if (!(dcx_data.length() == 0)) { out.write(dcx_data); } else { // Should not do this!! out.write("<srw:record>"); out.write("<srw:recordPacking>xml</srw:recordPacking>"); out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>"); out.write( "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" >"); StringBuffer result = new StringBuffer(""); construct_lucene_dc(result, resultDoc); out.write(result.toString()); out.write("</srw:recordData>"); out.write( "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } out.write("</srw_dc:dc>"); StringBuffer expand_data; boolean expand = false; if (content.startsWith("GGC-THES:AC:")) { String tmp_content = ""; tmp_content = content.replaceFirst("GGC-THES:AC:", ""); log.fatal("calling get"); expand_data = new StringBuffer( helpers.getExpand( "http://www.kbresearch.nl/general/lod_new/get/" + tmp_content + "?format=rdf", log)); log.fatal("get finini"); if (expand_data.toString().length() > 4) { out.write( "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); out.write(expand_data.toString()); expand = true; } } else { expand_data = new StringBuffer( helpers.getExpand( "http://www.kbresearch.nl/ANP.cgi?q=" + content, log)); if (expand_data.toString().length() > 0) { if (!expand) { out.write( "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >"); expand = true; } out.write(expand_data.toString()); } } if (expand) { out.write("</srw_dc:expand>"); } out.write("</srw:recordData>"); out.write( "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>"); out.write("</srw:record>"); } } } if ((do_err) || (numfound == 0)) { log.fatal("I haz suggestions"); try { spellq.setParam("spellcheck", true); spellq.setQueryType("/spell"); server = new CommonsHttpSolrServer(url); rsp = server.query(spellq); sdl = rsp.getResults(); SpellCheckResponse spell; spell = rsp.getSpellCheckResponse(); List<SpellCheckResponse.Suggestion> suggestions = spell.getSuggestions(); if (suggestions.isEmpty() == false) { suggest.append("<srw:extraResponseData>"); suggest.append("<suggestions>"); for (SpellCheckResponse.Suggestion sugg : suggestions) { suggest.append("<suggestionfor>" + sugg.getToken() + "</suggestionfor>"); for (String item : sugg.getSuggestions()) { suggest.append("<suggestion>" + item + "</suggestion>"); } suggest.append("</suggestions>"); suggest.append("</srw:extraResponseData>"); } do_sugg = true; } } catch (Exception e) { rsp = null; // log.fatal(e.toString()); } ; } ; if (!do_err) { if (facet != null) { try { fct = rsp.getFacetFields(); out.write("<srw:facets>"); for (String str : facet.split(",")) { out.write("<srw:facet>"); out.write("<srw:facetType>"); out.write(str); out.write("</srw:facetType>"); for (FacetField f : fct) { log.debug(f.getName()); // if (f.getName().equals(str+"_str") || (f.getName().equals(str+"_date")) ) { List<FacetField.Count> facetEnties = f.getValues(); for (FacetField.Count fcount : facetEnties) { out.write("<srw:facetValue>"); out.write("<srw:valueString>"); out.write(helpers.xmlEncode(fcount.getName())); out.write("</srw:valueString>"); out.write("<srw:count>"); out.write(Double.toString(fcount.getCount())); out.write("</srw:count>"); out.write("</srw:facetValue>"); // } } } out.write("</srw:facet>"); } out.write("</srw:facets>"); startRecord += 1; } catch (Exception e) { } // log.fatal(e.toString()); } } } else { out.write(diag); } out.write("</srw:records>"); // SearchRetrieve response footer String footer = this.SRW_FOOTER.replaceAll("\\$query", helpers.xmlEncode(query)); footer = footer.replaceAll("\\$startRecord", (startRecord).toString()); footer = footer.replaceAll("\\$maximumRecords", maximumRecords.toString()); footer = footer.replaceAll("\\$recordSchema", recordSchema); if (do_sugg) { out.write(suggest.toString()); } out.write(footer); } catch (MalformedURLException e) { out.write(e.getMessage()); } catch (IOException e) { out.write("TO ERR is Human"); } } } } out.close(); }
/** Main method of this example. */ @Override public void runExample(ExampleDriver driver) throws Exception { long startMs = System.currentTimeMillis(); CommandLine cli = driver.getCommandLine(); // Size of index batch requests to Solr // int batchSize = Integer.parseInt(cli.getOptionValue("batchSize", "500")); // Get a connection to Solr cloud using Zookeeper String zkHost = cli.getOptionValue("zkhost", ZK_HOST); String collectionName = cli.getOptionValue("collection", COLLECTION); int zkClientTimeout = Integer.parseInt(cli.getOptionValue("zkClientTimeout", "15000")); CloudSolrServer solr = new CloudSolrServer(zkHost); solr.setDefaultCollection(collectionName); solr.setZkClientTimeout(zkClientTimeout); solr.connect(); int numSent = 0; int numSkipped = 0; int lineNum = 0; SolrInputDocument doc = null; String line = null; // read file line-by-line BufferedReader reader = new BufferedReader(driver.readFile("log")); driver.rememberCloseable(reader); LogFormat fmt = LogFormat.valueOf(cli.getOptionValue("format", "solr")); // process each sighting as a document while ((line = reader.readLine()) != null) { doc = parseNextDoc(line, ++lineNum, fmt); if (doc != null) { addDocWithRetry(solr, doc, 10); ++numSent; } else { ++numSkipped; continue; } if (lineNum % 1000 == 0) log.info(String.format("Processed %d lines.", lineNum)); } // hard commit all docs sent solr.commit(true, true); float tookSecs = Math.round(((System.currentTimeMillis() - startMs) / 1000f) * 100f) / 100f; log.info( String.format( "Sent %d log messages (skipped %d) took %f seconds", numSent, numSkipped, tookSecs)); // queries to demonstrate results of indexing SolrQuery solrQuery = new SolrQuery("*:*"); solrQuery.setRows(0); QueryResponse resp = solr.query(solrQuery); SolrDocumentList hits = resp.getResults(); log.info("Match all docs distributed query found " + hits.getNumFound() + " docs."); solrQuery.set("shards", "shard1"); resp = solr.query(solrQuery); hits = resp.getResults(); log.info( "Match all docs non-distributed query to shard1 found " + hits.getNumFound() + " docs."); solrQuery.set("shards", "shard2"); resp = solr.query(solrQuery); hits = resp.getResults(); log.info( "Match all docs non-distributed query to shard2 found " + hits.getNumFound() + " docs."); solr.shutdown(); }
public void merge(ResponseBuilder rb, ShardRequest sreq) { // id to shard mapping, to eliminate any accidental dups HashMap<Object, String> uniqueDoc = new HashMap<>(); NamedList<Object> shardInfo = null; if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) { shardInfo = new SimpleOrderedMap<>(); rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo); } IndexSchema schema = rb.req.getSchema(); SchemaField uniqueKeyField = schema.getUniqueKeyField(); long numFound = 0; Float maxScore = null; boolean partialResults = false; List<ShardDoc> shardDocs = new ArrayList(); for (ShardResponse srsp : sreq.responses) { SolrDocumentList docs = null; if (shardInfo != null) { SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>(); if (srsp.getException() != null) { Throwable t = srsp.getException(); if (t instanceof SolrServerException) { t = ((SolrServerException) t).getCause(); } nl.add("error", t.toString()); StringWriter trace = new StringWriter(); t.printStackTrace(new PrintWriter(trace)); nl.add("trace", trace.toString()); if (srsp.getShardAddress() != null) { nl.add("shardAddress", srsp.getShardAddress()); } } else { docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response"); nl.add("numFound", docs.getNumFound()); nl.add("maxScore", docs.getMaxScore()); nl.add("shardAddress", srsp.getShardAddress()); } if (srsp.getSolrResponse() != null) { nl.add("time", srsp.getSolrResponse().getElapsedTime()); } shardInfo.add(srsp.getShard(), nl); } // now that we've added the shard info, let's only proceed if we have no error. if (srsp.getException() != null) { partialResults = true; continue; } if (docs == null) { // could have been initialized in the shards info block above docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response"); } NamedList<?> responseHeader = (NamedList<?>) srsp.getSolrResponse().getResponse().get("responseHeader"); if (responseHeader != null && Boolean.TRUE.equals( responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) { partialResults = true; } // calculate global maxScore and numDocsFound if (docs.getMaxScore() != null) { maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore()); } numFound += docs.getNumFound(); SortSpec ss = rb.getSortSpec(); Sort sort = ss.getSort(); NamedList sortFieldValues = (NamedList) (srsp.getSolrResponse().getResponse().get("merge_values")); NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema); List lst = (List) unmarshalledSortFieldValues.getVal(0); for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); Object id = doc.getFieldValue(uniqueKeyField.getName()); String prevShard = uniqueDoc.put(id, srsp.getShard()); if (prevShard != null) { // duplicate detected numFound--; // For now, just always use the first encountered since we can't currently // remove the previous one added to the priority queue. If we switched // to the Java5 PriorityQueue, this would be easier. continue; // make which duplicate is used deterministic based on shard // if (prevShard.compareTo(srsp.shard) >= 0) { // TODO: remove previous from priority queue // continue; // } } ShardDoc shardDoc = new ShardDoc(); shardDoc.id = id; shardDoc.shard = srsp.getShard(); shardDoc.orderInShard = i; Object scoreObj = lst.get(i); if (scoreObj != null) { shardDoc.score = ((Integer) scoreObj).floatValue(); } shardDocs.add(shardDoc); } // end for-each-doc-in-response } // end for-each-response Collections.sort( shardDocs, new Comparator<ShardDoc>() { @Override public int compare(ShardDoc o1, ShardDoc o2) { if (o1.score < o2.score) { return 1; } else if (o1.score > o2.score) { return -1; } else { return 0; // To change body of implemented methods use File | Settings | File // Templates. } } }); int resultSize = shardDocs.size(); Map<Object, ShardDoc> resultIds = new HashMap<>(); for (int i = 0; i < shardDocs.size(); i++) { ShardDoc shardDoc = shardDocs.get(i); shardDoc.positionInResponse = i; // Need the toString() for correlation with other lists that must // be strings (like keys in highlighting, explain, etc) resultIds.put(shardDoc.id.toString(), shardDoc); } // Add hits for distributed requests // https://issues.apache.org/jira/browse/SOLR-3518 rb.rsp.addToLog("hits", numFound); SolrDocumentList responseDocs = new SolrDocumentList(); if (maxScore != null) responseDocs.setMaxScore(maxScore); responseDocs.setNumFound(numFound); responseDocs.setStart(0); // size appropriately for (int i = 0; i < resultSize; i++) responseDocs.add(null); // save these results in a private area so we can access them // again when retrieving stored fields. // TODO: use ResponseBuilder (w/ comments) or the request context? rb.resultIds = resultIds; rb.setResponseDocs(responseDocs); if (partialResults) { rb.rsp .getResponseHeader() .add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE); } }
@Override public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) { ModifiableSolrParams solrParams = new ModifiableSolrParams(); QueryResponse response = null; RecommendResponse searchResponse = new RecommendResponse(); long step0ElapsedTime = 0; long step1ElapsedTime; List<String> recommendations = new ArrayList<String>(); try { // STEP 0 - get products from a user if (query.getUser() != null) { if (query.getProductIds() == null || query.getProductIds().size() == 0) { if (alreadyBoughtProducts != null) { query.setProductIds(alreadyBoughtProducts); } else { } } } solrParams = getInteractionsFromMeAndUSersThatILikedOrCommented(query.getUser()); response = SolrServiceContainer.getInstance() .getSocialActionService() .getSolrServer() .query(solrParams); step1ElapsedTime = response.getElapsedTime(); List<SocialAction> socialUsers = response.getBeans(SocialAction.class); if (socialUsers.size() == 0) { searchResponse.setNumFound(0); searchResponse.setResultItems(recommendations); searchResponse.setElapsedTime(-1); return searchResponse; } SocialAction currentUserInteractions = socialUsers.get(0); if (currentUserInteractions.getUserId().equals(query.getUser())) { socialUsers.remove(0); } else { currentUserInteractions = null; } final Map<String, Integer> userInteractionMap = new HashMap<String, Integer>(); if (currentUserInteractions != null) { List<String> usersThatPostedASnapshopToMe = currentUserInteractions.getUsersThatPostedASnapshopToMe(); if (usersThatPostedASnapshopToMe != null) { fillInteractions(usersThatPostedASnapshopToMe, userInteractionMap); } } for (SocialAction socialUser : socialUsers) { Integer userInteraction = userInteractionMap.get(socialUser.getUserId()); if (userInteraction == null) { userInteraction = 0; } if (socialUser.getUsersThatPostedASnapshopToMe() != null) { userInteraction += Collections.frequency(socialUser.getUsersThatPostedASnapshopToMe(), query.getUser()); } userInteractionMap.put(socialUser.getUserId(), userInteraction); } Comparator<String> interactionCountComparator = new Comparator<String>() { @Override public int compare(String a, String b) { if (userInteractionMap.get(a) > userInteractionMap.get(b)) { return -1; } else if (userInteractionMap.get(a).equals(userInteractionMap.get(b))) { return 0; } else { return 1; } } }; TreeMap<String, Integer> sorted_map = new TreeMap<String, Integer>(interactionCountComparator); sorted_map.putAll(userInteractionMap); solrParams = getSTEP2Params(query, maxReuslts, sorted_map); // TODO Facet for confidence value response = SolrServiceContainer.getInstance().getResourceService().getSolrServer().query(solrParams); // fill response object List<Resource> beans = response.getBeans(Resource.class); searchResponse.setResultItems(RecommendationQueryUtils.extractRecommendationIds(beans)); searchResponse.setElapsedTime( step0ElapsedTime + step1ElapsedTime + response.getElapsedTime()); SolrDocumentList docResults = response.getResults(); searchResponse.setNumFound(docResults.getNumFound()); } catch (Exception e) { System.out.println(solrParams); e.printStackTrace(); searchResponse.setNumFound(0); searchResponse.setResultItems(recommendations); searchResponse.setElapsedTime(-1); } return searchResponse; }
public static serverObjects respond( @SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); final Segment segment = sb.index; final SolrConnector connector = segment.fulltext().getDefaultConnector(); // avoid UNRESOLVED PATTERN prop.put("url", ""); prop.put("citations", 0); prop.put("sentences", 0); DigestURL uri = null; String url = ""; String hash = ""; int ch = 10; boolean filter = false; // show cited sentences only if (post != null) { if (post.containsKey("url")) { url = post.get("url"); if (!url.startsWith("http://") && !url.startsWith("https://") && !url.startsWith("ftp://") && !url.startsWith("smb://") && !url.startsWith("file://")) { url = "http://" + url; } } if (post.containsKey("hash")) { hash = post.get("hash"); } if (post.containsKey("ch")) { ch = post.getInt("ch", ch); } filter = post.getBoolean("filter"); } prop.put("filter", filter); if (url.length() > 0) { try { uri = new DigestURL(url, null); hash = ASCII.String(uri.hash()); } catch (final MalformedURLException e) { } } if (uri == null && hash.length() > 0) { try { uri = sb.getURL(ASCII.getBytes(hash)); if (uri == null) { connector.commit(true); // try again, that url can be fresh uri = sb.getURL(ASCII.getBytes(hash)); } } catch (IOException e) { ConcurrentLog.logException(e); } } if (uri == null) return prop; // no proper url addressed url = uri.toNormalform(true); prop.put("url", url); // get the document from the index SolrDocument doc; try { doc = segment .fulltext() .getDefaultConnector() .getDocumentById( hash, CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName()); } catch (final IOException e1) { return prop; } @SuppressWarnings("unchecked") ArrayList<String> title = (ArrayList<String>) doc.getFieldValue(CollectionSchema.title.getSolrFieldName()); String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); ArrayList<String> sentences = new ArrayList<String>(); if (title != null) for (String s : title) if (s.length() > 0) sentences.add(s); if (text != null && !text.isEmpty()) { SentenceReader sr = new SentenceReader(text); StringBuilder line; while (sr.hasNext()) { line = sr.next(); if (line.length() > 0) sentences.add(line.toString()); } } // for each line make a statistic about the number of occurrences somewhere else OrderedScoreMap<String> scores = new OrderedScoreMap<String>(null); // accumulates scores for citating urls LinkedHashMap<String, Set<DigestURL>> sentenceOcc = new LinkedHashMap<String, Set<DigestURL>>(); for (String sentence : sentences) { if (sentence == null || sentence.length() < 40) { // do not count the very short sentences sentenceOcc.put(sentence, null); continue; } try { sentence = sentence.replace('"', '\''); SolrDocumentList doclist = connector.getDocumentListByQuery( "text_t:\"" + sentence + "\"", CollectionSchema.url_chars_i.getSolrFieldName() + " asc", 0, 100, CollectionSchema.sku.getSolrFieldName()); int count = (int) doclist.getNumFound(); if (count > 0) { Set<DigestURL> list = new TreeSet<DigestURL>(); for (SolrDocument d : doclist) { String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName()); if (u == null || u.equals(url)) continue; scores.inc(u); try { list.add(new DigestURL(u, null)); } catch (final MalformedURLException e) { } } sentenceOcc.put(sentence, list); } } catch (final Throwable ee) { } } sentences.clear(); // we do not need this again // iterate the sentences int i = 0; int sentenceNr = 0; for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) { Set<DigestURL> app = se.getValue(); if (filter) { // prepare list, only include sentence with citation if (app != null && app.size() > 0) { StringBuilder dd = new StringBuilder(se.getKey()); prop.put("sentences_" + i + "_dt", sentenceNr); dd.append("<br/>appears in:"); for (DigestURL u : app) { if (u != null) { dd.append(" <a href=\"") .append(u.toNormalform(false)) .append("\">") .append(u.getHost()) .append("</a>"); } } prop.put("sentences_" + i + "_dd", dd.toString()); i++; } } else { // prepare list, include all sentences StringBuilder dd = new StringBuilder(se.getKey()); prop.put("sentences_" + i + "_dt", sentenceNr); if (app != null && app.size() > 0) { dd.append("<br/>appears in:"); for (DigestURL u : app) { if (u != null) { dd.append(" <a href=\"") .append(u.toNormalform(false)) .append("\">") .append(u.getHost()) .append("</a>"); } } } prop.put("sentences_" + i + "_dd", dd.toString()); i++; } sentenceNr++; } prop.put("sentences", i); // iterate the citations in order of number of citations i = 0; for (String u : scores.keyList(false)) { try { DigestURL uu = new DigestURL(u, null); prop.put("citations_" + i + "_dt", "<a href=\"" + u + "\">" + u + "</a>"); StringBuilder dd = new StringBuilder(); dd.append("makes ") .append(Integer.toString(scores.get(u))) .append(" citations: of ") .append(url); for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) { Set<DigestURL> occurls = se.getValue(); if (occurls != null && occurls.contains(uu)) dd.append("<br/><a href=\"/solr/select?q=text_t:%22") .append(se.getKey().replace('"', '\'')) .append("%22&rows=100&grep=&wt=grephtml\">") .append(se.getKey()) .append("</a>"); } prop.put("citations_" + i + "_dd", dd.toString()); i++; } catch (final MalformedURLException e) { } } prop.put("citations", i); // find similar documents from different hosts i = 0; for (String u : scores.keyList(false)) { if (scores.get(u) < ch) continue; try { DigestURL uu = new DigestURL(u, null); if (uu.getOrganization().equals(uri.getOrganization())) continue; prop.put("similar_links_" + i + "_url", u); i++; } catch (final MalformedURLException e) { } } prop.put("similar_links", i); prop.put("similar", i > 0 ? 1 : 0); // return rewrite properties return prop; }
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) { SortSpec ss = rb.getSortSpec(); Sort sort = ss.getSort(); SortField[] sortFields = null; if (sort != null) sortFields = sort.getSort(); else { sortFields = new SortField[] {SortField.FIELD_SCORE}; } SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField(); // id to shard mapping, to eliminate any accidental dups HashMap<Object, String> uniqueDoc = new HashMap<Object, String>(); // Merge the docs via a priority queue so we don't have to sort *all* of the // documents... we only need to order the top (rows+start) ShardFieldSortedHitQueue queue; queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount()); long numFound = 0; Float maxScore = null; for (ShardResponse srsp : sreq.responses) { SolrDocumentList docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response"); // calculate global maxScore and numDocsFound if (docs.getMaxScore() != null) { maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore()); } numFound += docs.getNumFound(); NamedList sortFieldValues = (NamedList) (srsp.getSolrResponse().getResponse().get("sort_values")); // go through every doc in this response, construct a ShardDoc, and // put it in the priority queue so it can be ordered. for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); Object id = doc.getFieldValue(uniqueKeyField.getName()); String prevShard = uniqueDoc.put(id, srsp.getShard()); if (prevShard != null) { // duplicate detected numFound--; // For now, just always use the first encountered since we can't currently // remove the previous one added to the priority queue. If we switched // to the Java5 PriorityQueue, this would be easier. continue; // make which duplicate is used deterministic based on shard // if (prevShard.compareTo(srsp.shard) >= 0) { // TODO: remove previous from priority queue // continue; // } } ShardDoc shardDoc = new ShardDoc(); shardDoc.id = id; shardDoc.shard = srsp.getShard(); shardDoc.orderInShard = i; Object scoreObj = doc.getFieldValue("score"); if (scoreObj != null) { if (scoreObj instanceof String) { shardDoc.score = Float.parseFloat((String) scoreObj); } else { shardDoc.score = (Float) scoreObj; } } shardDoc.sortFieldValues = sortFieldValues; queue.insertWithOverflow(shardDoc); } // end for-each-doc-in-response } // end for-each-response // The queue now has 0 -> queuesize docs, where queuesize <= start + rows // So we want to pop the last documents off the queue to get // the docs offset -> queuesize int resultSize = queue.size() - ss.getOffset(); resultSize = Math.max(0, resultSize); // there may not be any docs in range Map<Object, ShardDoc> resultIds = new HashMap<Object, ShardDoc>(); for (int i = resultSize - 1; i >= 0; i--) { ShardDoc shardDoc = (ShardDoc) queue.pop(); shardDoc.positionInResponse = i; // Need the toString() for correlation with other lists that must // be strings (like keys in highlighting, explain, etc) resultIds.put(shardDoc.id.toString(), shardDoc); } SolrDocumentList responseDocs = new SolrDocumentList(); if (maxScore != null) responseDocs.setMaxScore(maxScore); responseDocs.setNumFound(numFound); responseDocs.setStart(ss.getOffset()); // size appropriately for (int i = 0; i < resultSize; i++) responseDocs.add(null); // save these results in a private area so we can access them // again when retrieving stored fields. // TODO: use ResponseBuilder (w/ comments) or the request context? rb.resultIds = resultIds; rb._responseDocs = responseDocs; }
@RequestMapping(value = "/searchtweetresponse", method = RequestMethod.GET) @ResponseBody public String getTweets( @RequestParam("search") String searchKey, @RequestParam("start") int start, @RequestParam("end") int end) { logger.info("Welcome to Tweets"); JSONObject finalObject = new JSONObject(); int startRow = 0; int endRow = 10; // String tweetType="*"; if (start > startRow) { startRow = start; } if (end > endRow) { endRow = end; } SolrQuery query = new SolrQuery("tweet_content:*" + searchKey + "*"); query.setStart(startRow); query.setRows(endRow); HomeBean bean = new HomeBean(); Collection<JSONObject> objectList = new ArrayList<JSONObject>(); try { QueryResponse response = homeservice.getServiceResponse(query); if (response != null) { SolrDocumentList responseList = response.getResults(); long totalCount = responseList.getNumFound(); if (totalCount > 0) { bean.setResultCount(totalCount); } finalObject.put("total", totalCount); for (SolrDocument document : responseList) { JSONObject json = new JSONObject(); json.put("id", document.getFieldValue("id")); json.put("tweet", document.getFieldValue("tweet_content")); objectList.add(json); } } } catch (Exception e) { e.printStackTrace(); } finalObject.put("tweetList", objectList); query.set("facet", true); query.addFacetField("tweetPostedTime"); query.addSort("tweetPostedTime", SolrQuery.ORDER.asc); query.setRows(0); QueryResponse response = homeservice.getServiceResponse(query); // 2015-07-30T12:10:31Z DateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); SimpleDateFormat newFormat = new SimpleDateFormat("MM-dd-yyyy"); DateFormat dateFormat2 = new SimpleDateFormat("E MMM d hh:mm:ss zzz yyyy"); List<Date> myList = new ArrayList<Date>(); // Collection<JSONObject> tempList = new ArrayList<JSONObject>(); Collection<JSONObject> dateCountList = new ArrayList<JSONObject>(); JSONObject tempObj = new JSONObject(); try { if (response != null) { FacetField timeFacet = response.getFacetField("tweetPostedTime"); List<Count> list = timeFacet.getValues(); Iterator itr = list.iterator(); // Collection<String> productKeys = new ArrayList<String>(); // Collection<String> smartPhoneKeys = new ArrayList<String>(); while (itr.hasNext()) { Count timeObj = (Count) itr.next(); if (timeObj != null) { String timeName = timeObj.getName(); String originalTime = timeName.split("T")[0]; Date beforeParse = dateFormat1.parse(originalTime); myList.add(beforeParse); JSONObject timeObject = new JSONObject(); timeObject.put("date", newFormat.format(beforeParse)); timeObject.put("count", timeObj.getCount()); // tempList.add(timeObject); tempObj.put(newFormat.format(beforeParse), timeObject); } } } Collections.sort(myList); Iterator<Date> itr = myList.iterator(); while (itr.hasNext()) { Date inputDate = dateFormat2.parse(itr.next().toString()); String tempKey = newFormat.format(inputDate); System.out.println(tempKey); JSONObject newObj = (JSONObject) tempObj.get(tempKey); dateCountList.add(newObj); } finalObject.put("dateList", dateCountList); } catch (Exception e) { logger.error(e.getMessage()); e.printStackTrace(); } logger.info("bye Tweets"); return finalObject.toString(); }