@Override public void run() { Random random = LuceneTestCase.random(); int numSearches = 0; while (true && !stop) { numSearches++; try { // to come to the aid of their country. cloudClient.query(new SolrQuery(QUERIES[random.nextInt(QUERIES.length)])); } catch (Exception e) { System.err.println("QUERY REQUEST FAILED:"); e.printStackTrace(); if (e instanceof SolrServerException) { System.err.println("ROOT CAUSE:"); ((SolrServerException) e).getRootCause().printStackTrace(); } queryFails.incrementAndGet(); } try { Thread.sleep(random.nextInt(4000) + 300); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } log.info("num searches done:" + numSearches + " with " + queryFails + " fails"); }
private SearchEngineResponse<MediaItem> search(SolrQuery query) { SearchEngineResponse<MediaItem> response = new SearchEngineResponse<MediaItem>(); QueryResponse rsp; try { rsp = server.query(query); } catch (SolrServerException e) { e.printStackTrace(); Logger.getRootLogger().info(e.getMessage()); return null; } List<SolrMediaItem> solrItems = rsp.getBeans(SolrMediaItem.class); List<MediaItem> mediaItems = new ArrayList<MediaItem>(); for (SolrMediaItem solrMediaItem : solrItems) { try { MediaItem mediaItem = solrMediaItem.toMediaItem(); String id = mediaItem.getId(); mediaItem.setId(id); mediaItems.add(mediaItem); } catch (MalformedURLException ex) { Logger.getRootLogger().error(ex.getMessage()); } } response.setResults(mediaItems); return response; }
public void run() { // create new collections rapid fire Map<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>(); int cnt = random().nextInt(TEST_NIGHTLY ? 13 : 3) + 1; for (int i = 0; i < cnt; i++) { String collectionName = "awholynewstresscollection_" + name + "_" + i; int numShards = _TestUtil.nextInt(random(), 0, shardCount * 2) + 1; int replicationFactor = _TestUtil.nextInt(random(), 0, 3) + 1; int maxShardsPerNode = (((numShards * 2 * replicationFactor) / getCommonCloudSolrServer() .getZkStateReader() .getClusterState() .getLiveNodes() .size())) + 1; CloudSolrServer client = null; try { if (i == 0) { client = createCloudClient(null); } else if (i == 1) { client = createCloudClient(collectionName); } createCollection( collectionInfos, collectionName, numShards, replicationFactor, maxShardsPerNode, client, null, "conf1"); // remove collection ModifiableSolrParams params = new ModifiableSolrParams(); params.set("action", CollectionAction.DELETE.toString()); params.set("name", collectionName); QueryRequest request = new QueryRequest(params); request.setPath("/admin/collections"); if (client == null) { client = createCloudClient(null); } client.request(request); } catch (SolrServerException e) { e.printStackTrace(); throw new RuntimeException(e); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } finally { if (client != null) client.shutdown(); } } }
public SearchResponse search(SearchRequest request) { SolrQuery query = new SolrQuery(); query.setRows(request.getLimit()); query.setStart(request.getOffset() * request.getLimit()); BooleanQuery aggregate = new BooleanQuery(); if (!StringUtils.isBlank(request.getText())) { TermQuery termQuery = new TermQuery(new Term("", request.getText())); aggregate.add(termQuery, BooleanClause.Occur.SHOULD); } if (!StringUtils.isBlank(request.getEventId())) { aggregate.add( new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST); } query.setQuery(aggregate.toString()); log.info("QUERY IS: " + query.toString()); try { QueryResponse queryResponse = solrServer.query(query); log.info("RESPONSE FROM QUERY WAS: " + queryResponse); SolrDocumentList results = queryResponse.getResults(); log.info("RESULTS WAS: " + results); ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>(); for (SolrDocument result : results) { hits.add( new SearchResponse.Hit( ObjectType.session, String.valueOf(result.getFieldValue("id")), null)); } return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits); } catch (SolrServerException e) { e.printStackTrace(); } return null; }
public static void addDoc(String docnum, String payloads) { // 创建doc文档 SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", docnum); doc.addField("payloads", payloads); System.out.println(); System.out.println("docnum: " + docnum); System.out.println("payloads: " + payloads); log.info("docnum: " + docnum + "\tpayloads:" + payloads + "\twordcount:" + (j + existWords)); try { // 添加一个doc文档 UpdateResponse response = server.add(doc); // commit后才保存到索引库 server.commit(); /* System.out.println("Query Time:" + response.getQTime()); System.out.println("Elapsed Time:" + response.getElapsedTime()); System.out.println("Status:" + response.getStatus());*/ } catch (SolrServerException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
@Override public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) { productsToFilter = new ArrayList<String>(); String queryString = "*:*"; String sortCriteria = "user_count_purchased desc"; ModifiableSolrParams solrParams = new ModifiableSolrParams(); QueryResponse response = null; RecommendResponse searchResponse = new RecommendResponse(); try { String filterQueryString = RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter); if (query.getProductIds() != null && query.getProductIds().size() > 0) { productsToFilter.addAll(query.getProductIds()); } if (query.getUser() != null) { if (alreadyBoughtProducts != null) { productsToFilter.addAll(alreadyBoughtProducts); } } if (productsToFilter != null && productsToFilter.size() > 0) { if (filterQueryString.length() > 0) { filterQueryString += " OR "; } filterQueryString += RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter); } solrParams.set("q", queryString); solrParams.set("fq", filterQueryString); solrParams.set("sort", sortCriteria); solrParams.set("rows", maxReuslts); response = SolrServiceContainer.getInstance() .getRecommendService() .getSolrServer() .query(solrParams); // fill response object List<String> extractedRecommendations = RecommendationQueryUtils.extractRecommendationIds( response.getBeans(CustomerAction.class)); searchResponse.setResultItems(extractedRecommendations); searchResponse.setElapsedTime(response.getElapsedTime()); SolrDocumentList docResults = response.getResults(); searchResponse.setNumFound(docResults.getNumFound()); } catch (SolrServerException e) { e.printStackTrace(); searchResponse.setNumFound(0); searchResponse.setResultItems(new ArrayList<String>()); searchResponse.setElapsedTime(-1); } return searchResponse; }
public JSONArray Search(String query, String sort) { SolrServer server; SolrDocumentList docs = null; String Date_format = null; // for Date格式轉換 JSONArray jsonArray = new JSONArray(); try { server = new HttpSolrServer(url); // add SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(query); solrQuery.setStart(_start); solrQuery.setRows(_nbDocuments); solrQuery.setRequestHandler("query"); solrQuery.set("fl", "*,score"); // 設定fl參數,指定要傳回哪個field的資料,這裡設定所有field與score if (_fq1 != "") { solrQuery.addFilterQuery("ProductName:" + _fq1); } if (_fq2 != "") { solrQuery.addFilterQuery("publishedDate:" + _fq2); } if (sort != null) { solrQuery.addSortField(sort, ORDER.asc); } solrQuery.setRequestHandler("/browse"); response = server.query(solrQuery); docs = response.getResults(); if (docs != null) { System.out.println( docs.getNumFound() + " documents found, " + docs.size() + " returned : "); setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端 for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); JSONObject jsonObject = new JSONObject(); for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) { Entry<String, Object> entry = it2.next(); if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串,方便前端呈現 Date_format = entry.getValue().toString(); jsonObject.put(entry.getKey(), Date_format); } else { // 一般情況 jsonObject.put(entry.getKey(), entry.getValue()); } } System.out.print("\n"); // 將總共找到幾筆資料存在jsonarray的最後面傳給前端 jsonObject.put("TotalResultFound", docs.getNumFound()); jsonArray.add(jsonObject); } } } catch (SolrServerException e) { // TODO Auto-generated catch block e.printStackTrace(); } return jsonArray; }
@Override protected OntologyTerm createRootTerm(OntologySearcher ontologySearcher) { try { return new LazyLoadedOntologyTerm(ontologySearcher, "MI:0190", "Interaction Type"); } catch (SolrServerException e) { e.printStackTrace(); addErrorMessage("Could not load the tree", ""); } return null; }
public void forceCommitPending() { try { server.commit(); } catch (SolrServerException ex) { ex.printStackTrace(); Logger.getRootLogger().error(ex.getMessage()); } catch (IOException ex) { ex.printStackTrace(); Logger.getRootLogger().error(ex.getMessage()); } }
private void index(TruckEvent truckEvent) { try { LOG.info("Starting process to solr index document[" + truckEvent + "]"); UpdateResponse response = server.addBean(truckEvent); LOG.info( "Indexed document with id: " + truckEvent.getId() + " status: " + response.getStatus()); } catch (IOException e) { LOG.error("Could not index document: " + truckEvent.getId() + " " + e.getMessage()); e.printStackTrace(); } catch (SolrServerException e) { LOG.error("Could not index document: " + truckEvent.getId() + " " + e.getMessage()); e.printStackTrace(); } }
public void run() { while (true) { try { Thread.sleep(15000); server.commit(); LOG.info("Committing Index"); } catch (InterruptedException e) { LOG.error("Interrupted: " + e.getMessage()); e.printStackTrace(); } catch (SolrServerException e) { LOG.error("Error committing: " + e.getMessage()); e.printStackTrace(); } catch (IOException e) { LOG.error("Error committing: " + e.getMessage()); e.printStackTrace(); } } }
/** * Returns a MODS record for a given recordIdentifier. * * @param id a recordIdentifier for a solr document * @return the ModsType for this recordidentifier * @see ModsType */ public ModsType getMods(String id) throws JAXBException { SolrDocumentList docs; SolrDocument doc; ModsType modsType = new ModsType(); HttpSolrServer server = null; try { server = SolrServer.getSolrConnection(); SolrQuery query = new SolrQuery("recordIdentifier:" + id); QueryResponse response = server.query(query); docs = response.getResults(); if (docs.size() == 0) throw new ResourceNotFoundException("Item " + id + " not found"); else { doc = docs.get(0); modsType = getModsType(doc); } } catch (SolrServerException se) { se.printStackTrace(); log.error(se.getMessage()); } return modsType; }
// deprecated - use getModsType instead, rewrite and use this if we decide to wrap // the mods document in an <item> tag public Item getItem(String id) { SolrDocumentList docs; SolrDocument doc; Item item = new Item(); HttpSolrServer server = null; try { server = SolrServer.getSolrConnection(); SolrQuery query = new SolrQuery("recordIdentifier:" + id); QueryResponse response = server.query(query); docs = response.getResults(); if (docs.size() == 0) item = null; else { doc = docs.get(0); item = setItem(doc); } } catch (SolrServerException se) { log.error(se.getMessage()); se.printStackTrace(); } return item; }
public boolean execute() { SolrQuery query = queryGenerator.generate(); try { logger.debug("executing query: " + query); long init = System.nanoTime(); QueryResponse response = this.executeQuery(query); long clientTime = (System.nanoTime() - init) / 1000000; logger.debug( response.getResults().getNumFound() + " results found in " + response.getQTime() + " ms"); if (response.getQTime() < 0) { throw new RuntimeException( "The query returned less than 0 as q time: " + response.getResponseHeader().get("q") + response.getQTime()); } executor.notifyQueryExecuted(response, clientTime); } catch (SolrServerException e) { logger.error("Error on Query " + query); e.printStackTrace(); executor.notifyError(new QueryException(e, query)); return false; } return true; }
public void startElement(XmlPullParser parser, String wholeDoc) throws Exception { String qName = parser.getName(); if (qName.equalsIgnoreCase("DOC")) { if (currentDoc != null) { throw new Exception("Nested Document taggs. Input file has problems.\n"); } else { currentDoc = new SolrInputDocument(); switch (parser.getAttributeCount()) { case 0: category = "web"; break; case 1: category = "discussion"; baseID = parser.getAttributeValue(0); currentDoc.addField("id", parser.getAttributeValue(0)); break; case 2: category = "news"; currentDoc.addField("id", parser.getAttributeValue(0)); // TODO: process news type attribute if needed break; } currentDoc.addField("category", category); currentDoc.addField("whole_text", wholeDoc); if (category.equals("web")) { // TODO: call clean } /*Object[] preprocess = Preprocessor.Tokenize(StripXMLTags.strip(wholeDoc).toString()); currentDoc.addField("offsets", preprocess[0]); currentDoc.addField("tokens", preprocess[1]); currentDoc.addField("tree", preprocess[2]);*/ } } else if (qName.equalsIgnoreCase("DOCID")) { // Should happen in web documents only inDocId = true; } else if (qName.equalsIgnoreCase("DOCTYPE")) { // Should happen in web documents only // TODO: process if this element is needed // inDocType = true; } else if (qName.equalsIgnoreCase("DATETIME")) { inDate = true; } else if (qName.equalsIgnoreCase("DATELINE")) { inDate = true; } else if (qName.equalsIgnoreCase("POSTDATE")) { inDate = true; } else if (qName.equalsIgnoreCase("P")) { inContent = true; } else if (qName.equalsIgnoreCase("headline")) { inContent = true; } else if (qName.equalsIgnoreCase("post")) { assert (category.equals("discussion") || category.equals("web")); assert (baseID != null); if (currentDoc != null) { // Finish the previous doc if (category.equals("web") && currentDoc.getFieldValues("id") == null) { currentDoc.addField("id", baseID); } assert (currentDoc.getFieldValue("id") != null); try { SolrImporter.addDoc(server, currentDoc); /*//Make sure offsets are correct String rawText = SolrInterface.getRawDocument(baseID); int diff = StripXMLTags.verify(wholeDoc, new StringBuffer(rawText)); if (diff!=-1){ System.out.println(diff+":'("); }*/ } catch (SolrServerException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } currentDoc = null; } currentDoc = new SolrInputDocument(); currentDoc.addField("category", category); if (parser.getAttributeCount() == 3 && parser.getAttributeName(0).equalsIgnoreCase("author")) { assert (category.equals("discussion")); currentDoc.addField("author", parser.getAttributeValue(0)); // Append post id to doc id currentDoc.addField("id", baseID + "." + parser.getAttributeValue(2)); } else { assert (category.equals("web")); // Append the poster number to the doc id currentDoc.addField("id", baseID + "." + (count++)); } // TODO: datetime for discussions if needed inContent = true; } else if (qName.equalsIgnoreCase("POSTER")) { inPoster = true; } else if (qName.equalsIgnoreCase("BODY")) { // ignore since we populate fields inside } else if (qName.equalsIgnoreCase("TEXT")) { // ignore since we populate fields inside } else if (qName.equalsIgnoreCase("docs")) { // ignore } else if (qName.equalsIgnoreCase("QUOTE")) { // Note: Hack so that we have the things before the quote tag due to not XML compliant quote // tag crashing parser /*if (category.equalsIgnoreCase("web")){ //currentDoc.addField("content", contentBuffer.toString()); endElement("post"); }*/ } else if (qName.equals("a")) { // ignore } else if (qName.equals("img")) { // ignore } else { System.out.println("Unexpected node:" + qName); } }
public NGDItem call() { NGDItem totalResults = null; SolrQuery query = new SolrQuery(); query.setQuery("+text:(*:*)"); query.addFilterQuery("+pub_date_year:[1993 TO 2013]"); query.setParam("fl", "pmid"); for (int i = 0; i < term1Array.size(); i++) { String term1Combined = createCombinedTerm(term1Array.get(i)); query.addFilterQuery("+text:(" + term1Combined + ")"); } for (int j = 0; j < term2Array.size(); j++) { String term2Combined = createCombinedTerm(term2Array.get(j)); query.addFilterQuery("+text:(" + term2Combined + ")"); } boolean retry = false; try { QueryResponse rsp = this.server.query(query); totalResults = new NGDItem( this.term1count, this.term2count, this.term1, this.term2, this.term1Array, this.term2Array, rsp.getResults().getNumFound(), useAlias, this.totalDocCount); } catch (SolrServerException e) { log.warning(e.getMessage()); e.printStackTrace(); retry = true; } if (retry) { log.warning("retrying query"); try { // retry once..... QueryResponse rsp = this.server.query(query); totalResults = new NGDItem( this.term1count, this.term2count, this.term1, this.term2, this.term1Array, this.term2Array, rsp.getResults().getNumFound(), useAlias, this.totalDocCount); } catch (SolrServerException e) { log.warning(e.getMessage()); e.printStackTrace(); System.exit(1); } } return totalResults; }