protected SolrDocumentList getSolrDocumentByUUIDs(List<String> uuids) { String operand = "OR"; SolrDocumentList solrDocumentList = null; StringBuffer sb = new StringBuffer(); sb.append("("); for (String uuid : uuids) { sb.append("("); sb.append(uuid); sb.append(")"); sb.append(operand); } String queryString = sb.substring(0, sb.length() - operand.length()) + ")"; SolrQuery query = new SolrQuery(); SolrDocument solrDocument = null; try { SolrServer server = SolrServerManager.getInstance().getSolrServer(); QueryResponse response = null; query.setQuery("id:" + queryString); response = server.query(query); solrDocumentList = response.getResults(); } catch (SolrServerException e) { LOG.info("Exception :", e); throw new DocstoreIndexException(); } return solrDocumentList; }
private static boolean commit(List<Review> rs) throws Exception { if (server == null) { server = getServer(); } Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); for (Review r : rs) { SolrInputDocument doc1 = new SolrInputDocument(); for (Field f : r.getClass().getDeclaredFields()) { if (!f.getName().equals("content") && !f.getName().equals("title")) { doc1.addField(f.getName(), f.getLong(r), 1.0f); } else { doc1.addField(f.getName(), f.get(r), 1.0f); } } docs.add(doc1); if (counter++ % 10000 == 0) { System.out.println("Process " + counter); } } server.add(docs); // 提交 server.commit(); System.out.println("Commit @ " + counter); return true; }
public void commitRecordsToSolr(SolrServer solr) throws SolrServerException, IOException { if (commitCounter < MAX_WARM_SEARCH_COUNT) { solr.commit(false, false, false); // LOG.info("Time taken to commit with waitSearch = false commitCounter < // MAX_WARM_SEARCH_COUNT"); commitCounter++; } else { timeSinceLastCommit = System.currentTimeMillis() - lastCommitTime; if (timeSinceLastCommit > TIME_FOR_WARMING_SEARCHERS) { solr.commit(false, false, false); commitCounter++; // commitCounter = 1; // LOG.info("Time taken to commit with waitSearch = false timeSinceLastCommit > // TIME_FOR_WARMING_SEARCHERS"); } else { long startTime = System.currentTimeMillis(); solr.commit(false, true, false); long endTime = System.currentTimeMillis(); // LOG.info("Time taken to commit with waitSearch = true and time taken is " + (endTime - // startTime)); commitCounter = 1; } } lastCommitTime = System.currentTimeMillis(); }
@Test @Ignore // very incomplete public void testPathAnalysis() throws SolrServerException, IOException { SolrServer solr = getSolr(); // Not really unit testing the schema here because the path logic in the handler is too relevant // - could be switched to IndexingItemProgress p = mock(IndexingItemProgress.class); CmsRepository repo = new CmsRepository("http://ex.ampl:444/s/rep1"); when(p.getRepository()).thenReturn(repo); when(p.getRevision()).thenReturn(new RepoRevision(35L, new Date())); CmsChangesetItem item = mock(CmsChangesetItem.class); when(item.getPath()) .thenReturn(new CmsItemPath("/dir/doc main.xml")) .thenReturn(new CmsItemPath("/dir sect/doc appendix.xml")); ; IndexingDocIncrementalSolrj doc = new IndexingDocIncrementalSolrj(); when(p.getFields()).thenReturn(doc); when(p.getItem()).thenReturn(item); HandlerPathinfo handlerPathinfo = new HandlerPathinfo(); handlerPathinfo.handle(p); solr.add(doc.getSolrDoc()); fail( "need to test effects of analyzed path* fields and confirm the need for name and extension"); }
@Test(dependsOnMethods = "testAddDocument") public void testBasicQuery() throws SolrServerException { SolrServer server = solrServerFactory.getServer(); SolrParams params = new SolrQuery("*:*"); SolrDocumentList results = server.query(params).getResults(); assertTrue(results.getNumFound() > 0, "didn't return any results"); }
protected void indexSolrDocuments(List<SolrInputDocument> solrDocs, boolean isCommit) { synchronized (this.getClass()) { SolrServer solr = null; try { solr = SolrServerManager.getInstance().getSolrServer(); UpdateResponse response = solr.add(solrDocs); if (isCommit) { try { commitRecordsToSolr(solr); } catch (HttpSolrServer.RemoteSolrException e) { lastCommitTime = System.currentTimeMillis(); if (e.getMessage().startsWith(MAX_WARM_SEARCH)) { LOG.warn(e.getMessage()); } else { throw e; } } } } catch (Exception e) { LOG.info("Exception :", e); rollback(solr); throw new DocstoreIndexException(e); } } }
public List<DOSearchResult> getSearchResults( DORestSearchRequest request, SearchErrors errors, Map<String, ArrayList<String>> nerMap) { List<DOSearchResult> result = new ArrayList<DOSearchResult>(); QueryParam tcqp = null; QueryResponse qres = null; try { SolrServer server = solrConnectionUtils.getRestSolrServer(); tcqp = restQueryCreator.getSearchQuery(request, nerMap); qres = server.query(tcqp); if (qres != null) { DOSearchResult serachRes = null; if (request.isGrouprequest()) { serachRes = DOResponseUtils.processGroupQueryResponse( qres, Constants.RESPONSE_TYPE_REST, request.isSpellcheckApplied()); } else { serachRes = DOResponseUtils.processQueryResponse( qres, Constants.RESPONSE_TYPE_REST, request.isSpellcheckApplied()); } result.add(serachRes); } } catch (SolrServerException e) { logger.error(e.getMessage(), e); SearchError error = new SearchError(ErrorCode.SOLR_ERROR_CODE, e.getMessage()); errors.add(error); } catch (Exception e) { logger.error(e.getMessage(), e); SearchError error = new SearchError(ErrorCode.SOLR_ERROR_CODE, e.getMessage()); errors.add(error); } return result; }
@Test(dependsOnMethods = {"testGetServer"}) public void testBadQuery() throws SolrServerException { SolrServer server = solrServerFactory.getServer(); SolrParams params = new SolrQuery("text that is not found"); QueryResponse response = server.query(params); assertEquals(0L, response.getResults().getNumFound()); }
@Test public void fieldsRequestsShouldTurnIntoSelectors() throws Exception, TException { String table = "fieldsRequestsShouldTurnIntoSelectors"; SolrServer server = TestTableCreator.newTable(table) .withRowCount(1) .withRecordsPerRow(2) .withRecordColumns("fam.value", "fam.mvf") .create(); SolrQuery query = new SolrQuery("value0-0"); query.setFields("fam.value"); QueryResponse response = server.query(query); assertEquals( "We should get our doc back for a valid test.", 1l, response.getResults().getNumFound()); SolrDocument docResult = response.getResults().get(0); assertEquals("value0-0", docResult.getFieldValue("fam.value")); assertNull( "We shouldn't get this one back since it wasnt in our fields.", docResult.getFieldValues("fam.mvf")); removeTable(table); }
protected void indexAndDelete( List<SolrInputDocument> solrDocs, List<String> idsToDelete, boolean isCommit) { synchronized (this.getClass()) { SolrServer solr = null; try { solr = SolrServerManager.getInstance().getSolrServer(); if (CollectionUtils.isNotEmpty(solrDocs)) { UpdateResponse response = solr.add(solrDocs); } // deleting document which contains operation delete delete(idsToDelete, solr); if (isCommit) { try { commitRecordsToSolr(solr); } catch (HttpSolrServer.RemoteSolrException e) { lastCommitTime = System.currentTimeMillis(); if (e.getMessage().startsWith(MAX_WARM_SEARCH)) { LOG.warn(e.getMessage()); } else { throw e; } } } } catch (Exception e) { LOG.info("Exception :", e); rollback(solr); throw new DocstoreIndexException(e); } } }
/* (non-Javadoc) * @see fr.hoteia.qalingo.core.solr.service.ProductSkuSolrService#addOrUpdateProductSku(fr.hoteia.qalingo.core.domain.ProductSku) */ public void addOrUpdateProductSku( final ProductSku productSku, final MarketArea marketArea, final Retailer retailer) throws SolrServerException, IOException { if (productSku.getId() == null) { throw new IllegalArgumentException("Id cannot be blank or null."); } if (logger.isDebugEnabled()) { logger.debug("Indexing productSku " + productSku.getId()); logger.debug("Indexing productSku " + productSku.getBusinessName()); logger.debug("Indexing productSku " + productSku.getDescription()); logger.debug("Indexing productSku " + productSku.getCode()); } ProductSkuSolr productSkuSolr = new ProductSkuSolr(); productSkuSolr.setId(productSku.getId()); productSkuSolr.setBusinessname(productSku.getBusinessName()); productSkuSolr.setDescription(productSku.getDescription()); productSkuSolr.setCode(productSku.getCode()); ProductSkuPrice productSkuPrice = productSku.getPrice(marketArea.getId(), retailer.getId()); if (productSkuPrice != null) { BigDecimal salePrice = productSkuPrice.getSalePrice(); productSkuSolr.setPrice(salePrice.toString()); } productSkuSolrServer.addBean(productSkuSolr); productSkuSolrServer.commit(); }
@Test public void basicFullTextQuery() throws Exception { String table = "basicFullTextQuery"; SolrServer server = TestTableCreator.newTable(table) .withRowCount(1) .withRecordsPerRow(2) .withRecordColumns("fam.value", "fam.mvf", "fam.mvf") .create(); SolrQuery query = new SolrQuery("value0-0"); QueryResponse response = server.query(query); assertEquals("We should get our doc back.", 1l, response.getResults().getNumFound()); SolrDocument docResult = response.getResults().get(0); assertEquals("0", docResult.getFieldValue("recordid")); assertEquals("value0-0", docResult.getFieldValue("fam.value")); Collection<Object> mvfVals = docResult.getFieldValues("fam.mvf"); assertTrue( "We should get all our values back[" + mvfVals + "]", CollectionUtils.isEqualCollection(mvfVals, Lists.newArrayList("value0-0", "value0-0"))); removeTable(table); }
public static void main(String a[]) throws MalformedURLException, SolrServerException { SolrServer hotServer = new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds"); SolrQuery solrLocrQuery = new SolrQuery(); /*solrLocrQuery.setParam("pos", "ORB"); solrLocrQuery.setParam("locale", "en_US");*/ solrLocrQuery.setRows(300); QueryResponse queryLocResponse = hotServer.query(solrLocrQuery); SolrDocumentList locList = queryLocResponse.getResults(); for (SolrDocument doc : locList) { try { String str = doc.get("text_s").toString(); if (str.toLowerCase().contains("price")) { if (str.toLowerCase().contains("assurance")) { String id = doc.get("id").toString(); String url = doc.get("title_s").toString(); System.out.println(id + "|" + url); } } } catch (Exception e) { } } }
private static void createSolrAlert(String userId, String created, String hashtag) { Date d = new Date(created); // SimpleDateFormat formatter=new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z+9HOUR'"); // SimpleDateFormat formatter=new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z-4HOUR'"); SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); created = formatter.format(d); SolrServer server = new HttpSolrServer("http://sandbox.hortonworks.com:8983/solr/tweets"); // create a tweet doc SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", userId + ":" + hashtag + "-" + created); doc.addField("doctype_s", "alert"); doc.addField("createdAt_dt", created); doc.addField("tag_s", hashtag); try { server.add(doc); System.out.println("SolrBolt: successfully added alert to Solr server" + hashtag); } catch (Exception e) { e.printStackTrace(); // collector.fail(input); } }
/** * Query the indexed csv data using Tabular query, with the parameter {@link SirenParams#TQ}. * * @throws SolrServerException */ public void query() throws SolrServerException { System.out.println("Query: [0]'conference1'"); final SolrQuery query1 = new SolrQuery(); query1.setQueryType("siren"); query1.set(SirenParams.TQ, "[0]'conference1'"); // tabular query final QueryResponse response1 = server.query(query1); for (SolrDocument d : response1.getResults()) { System.out.println(d); } System.out.println(); System.out.println("Query: * <name> 'john AND gartner'"); final SolrQuery query2 = new SolrQuery(); query2.setQueryType("siren"); query2.set(SirenParams.NQ, "* <name> 'john AND gartner'"); // ntriple query final QueryResponse response2 = server.query(query2); for (SolrDocument d : response2.getResults()) { System.out.println(d); } System.out.println(); System.out.println("Query: * <type> <student> AND [2]'[* TO 2010]'^^<xsd:int>"); final SolrQuery query3 = new SolrQuery(); query3.setQueryType("siren"); query3.set(SirenParams.NQ, "* <type> <student>"); // ntriple query query3.set(SirenParams.TQ, "[2]'[* TO 2010]'^^<xsd:int>"); // tabular query final QueryResponse response3 = server.query(query3); for (SolrDocument d : response3.getResults()) { System.out.println(d); } System.out.println(); }
@Test(dependsOnMethods = "testBasicQuery", dataProvider = "documentMap") public void testIdQuery(Map<String, String[]> document) throws SolrServerException { String id = document.get("id")[0]; SolrServer server = solrServerFactory.getServer(); SolrParams params = new SolrQuery("id:" + id); SolrDocumentList results = server.query(params).getResults(); assertEquals(results.getNumFound(), 1, "didn't find article by id"); }
public JSONArray Search(String query, String sort) { SolrServer server; SolrDocumentList docs = null; String Date_format = null; // for Date格式轉換 JSONArray jsonArray = new JSONArray(); try { server = new HttpSolrServer(url); // add SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(query); solrQuery.setStart(_start); solrQuery.setRows(_nbDocuments); solrQuery.setRequestHandler("query"); solrQuery.set("fl", "*,score"); // 設定fl參數,指定要傳回哪個field的資料,這裡設定所有field與score if (_fq1 != "") { solrQuery.addFilterQuery("ProductName:" + _fq1); } if (_fq2 != "") { solrQuery.addFilterQuery("publishedDate:" + _fq2); } if (sort != null) { solrQuery.addSortField(sort, ORDER.asc); } solrQuery.setRequestHandler("/browse"); response = server.query(solrQuery); docs = response.getResults(); if (docs != null) { System.out.println( docs.getNumFound() + " documents found, " + docs.size() + " returned : "); setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端 for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); JSONObject jsonObject = new JSONObject(); for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) { Entry<String, Object> entry = it2.next(); if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串,方便前端呈現 Date_format = entry.getValue().toString(); jsonObject.put(entry.getKey(), Date_format); } else { // 一般情況 jsonObject.put(entry.getKey(), entry.getValue()); } } System.out.print("\n"); // 將總共找到幾筆資料存在jsonarray的最後面傳給前端 jsonObject.put("TotalResultFound", docs.getNumFound()); jsonArray.add(jsonObject); } } } catch (SolrServerException e) { // TODO Auto-generated catch block e.printStackTrace(); } return jsonArray; }
@Test public void testFilenameNumberedDelimiterDash() throws Exception { SolrServer solr = getSolr(); SolrInputDocument doc1 = new SolrInputDocument(); doc1.addField("id", "1"); doc1.addField("pathnamebase", "MAP-12345678"); solr.add(doc1); SolrInputDocument doc2 = new SolrInputDocument(); doc2.addField("id", "2"); doc2.addField("pathnamebase", "TOP-12345678"); solr.add(doc2); solr.commit(); assertEquals( "exact match", 1, solr.query(new SolrQuery("name:MAP-12345678")).getResults().getNumFound()); assertEquals( "exact match", 1, solr.query(new SolrQuery("name:TOP-12345678")).getResults().getNumFound()); // split on dash assertEquals( "split on dash", 1, solr.query(new SolrQuery("name:TOP")).getResults().getNumFound()); assertEquals( "split on dash", 2, solr.query(new SolrQuery("name:12345678")).getResults().getNumFound()); assertEquals( "split on dash", 1, solr.query(new SolrQuery("name:TOP 12345678")).getResults().getNumFound()); }
@Test public void testFulltextSearchNumbersHyphen() throws Exception { SolrServer solr = getSolr(); SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", "1"); doc.addField("text", "word The SD-500 product"); solr.add(doc); solr.commit(); assertEquals( "Should match simple word", 1, solr.query(new SolrQuery("text:word")).getResults().getNumFound()); assertEquals( "Should match exact product", 1, solr.query(new SolrQuery("text:SD-500")).getResults().getNumFound()); assertEquals( "Should match exact product lowercase", 1, solr.query(new SolrQuery("text:sd-500")).getResults().getNumFound()); // With hyphen also the StandardTokenizer will split. assertEquals( "Could match part 1", 1, solr.query(new SolrQuery("text:SD")).getResults().getNumFound()); assertEquals( "Could match part 2", 1, solr.query(new SolrQuery("text:500")).getResults().getNumFound()); assertEquals( "Could match quoted context", 1, solr.query(new SolrQuery("text:\"The SD-500 product\"")).getResults().getNumFound()); }
private SolrServer createServerAndTableWithSimpleTestDoc(String table) throws BlurException, TException, IOException, SolrServerException { createTable(table); SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table); SolrInputDocument doc; doc = createSimpleTestDoc(); server.add(doc); return server; }
/** * Execute a search query * * @param solrQuery The solr query to perform * @param hasFacet Indicates whether the solr query has facets * @return The search result * @throws SolrServerException */ private SolrSearchResult executeSearch(SolrQuery solrQuery, boolean hasFacet) throws SolrServerException { LOGGER.debug("Query to send to Solr: {}", solrQuery.toString()); SolrServer solrServer = SolrManager.getInstance().getSolrServer(); QueryResponse queryResponse = solrServer.query(solrQuery); if (hasFacet) { return new SolrSearchResult(queryResponse.getResults(), queryResponse.getFacetFields()); } else { return new SolrSearchResult(queryResponse.getResults()); } }
/** * delete * * <p>Set the status of the given object to deleted * * @param fedoraObject The fedora object * @throws FedoraClientException */ public void delete(FedoraObject fedoraObject) throws FedoraClientException { FedoraBroker.updateObjectState(fedoraObject.getObject_id(), "D"); // Add this as a work around for the fact that commitWithin does not seem to work for // the Solr XML delete so we want to commit after delete SolrServer solrServer = SolrManager.getInstance().getSolrServer(); try { solrServer.deleteById(fedoraObject.getObject_id(), 5000); } catch (IOException e) { LOGGER.debug("Exception committing delete", e); } catch (SolrServerException e) { LOGGER.debug("Exception committing delete", e); } }
public static void cache(String queryId, Object obj, SolrServer server, boolean commit) throws IOException, SolrServerException { SolrInputDocument newDoc = new SolrInputDocument(); newDoc.addField(Constants.CACHE_INDEX_idFieldName, queryId); newDoc.addField(Constants.CACHE_INDEX_valueFieldName, SerializableUtils.serializeBase64(obj)); // newDoc.addField(valueTextFieldName, ojbectToString(obj)); if (server == null) { throw new SolrServerException("server IS NULL!"); } server.add(newDoc); if (commit) server.commit(); }
@Test public void test2() throws Exception { SolrInputDocument doc = new SolrInputDocument(); // ID doc.setField("id", "3"); // 名称 doc.setField("name", "何灵"); // 添加 solrServer.add(doc); // 提交 solrServer.commit(); }
@Override public void delete(String id) { try { SolrServer server = SolrServerManager.getInstance().getSolrServer(); deleteRecordInSolr(server, id); server.commit(); } catch (SolrServerException e) { LOG.info("Exception :", e); throw new DocstoreIndexException(); } catch (IOException e) { LOG.info("Exception :", e); throw new DocstoreIndexException(); } }
public void delete(Object entity) { if (!(entity instanceof AbstractEntity)) { return; } try { solrServer.deleteById(((AbstractEntity) entity).getId()); solrServer.commit(); } catch (SolrServerException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testFulltextSearchEmail() throws Exception { SolrServer solr = getSolr(); SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", "1"); doc.addField("text", "contact [email protected]"); solr.add(doc); solr.commit(); assertEquals( "Should match simple word", 1, solr.query(new SolrQuery("text:contact")).getResults().getNumFound()); // Likely NOT possible with WDF in index pipeline. assertEquals( "Should match exact email", 1, solr.query(new SolrQuery("text:[email protected]")).getResults().getNumFound()); assertEquals( "Should match exact email - Quoted", 1, solr.query(new SolrQuery("text:\"[email protected]\"")).getResults().getNumFound()); assertEquals( "Could match part 1", 1, solr.query(new SolrQuery("text:support")).getResults().getNumFound()); assertEquals( "Could match part 2", 1, solr.query(new SolrQuery("text:example.com")).getResults().getNumFound()); }
@Test public void test1() throws Exception { String baseURL = "http://192.168.19.128:8080/solr"; SolrServer solr = new HttpSolrServer(baseURL); SolrInputDocument doc = new SolrInputDocument(); doc.setField("id", "2"); doc.setField("name", "kobe"); solr.add(doc); // 提交 solr.commit(); }
@Override // Input Map is expected to have the suffixes on the key names public void add(String type, Map<String, Object> properties) throws PersistenceException { LOGGER.debug("type = {}", type); if (type == null || type.isEmpty()) { return; } if (properties == null || properties.isEmpty() || properties.containsValue("guest")) { return; } LOGGER.debug("Adding entry of type {}", type); // Set Solr Core name to type and create/connect to Solr Core SolrServer coreSolrServer = getSolrCore(type); if (coreSolrServer == null) { return; } Date now = new Date(); // DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); // String createdDate = df.format(now); SolrInputDocument solrInputDocument = new SolrInputDocument(); solrInputDocument.addField("createddate_tdt", now); for (Map.Entry<String, Object> entry : properties.entrySet()) { solrInputDocument.addField(entry.getKey(), entry.getValue()); } try { UpdateResponse response = coreSolrServer.add(solrInputDocument); LOGGER.debug("UpdateResponse from add of SolrInputDocument: {}", response); } catch (SolrServerException e) { LOGGER.info("SolrServerException while adding Solr index for persistent type {}", type, e); doRollback(coreSolrServer, type); throw new PersistenceException( "SolrServerException while adding Solr index for persistent type " + type, e); } catch (IOException e) { LOGGER.info("IOException while adding Solr index for persistent type {}", type, e); doRollback(coreSolrServer, type); throw new PersistenceException( "IOException while adding Solr index for persistent type " + type, e); } catch (RuntimeException e) { LOGGER.info("RuntimeException while adding Solr index for persistent type {}", type, e); doRollback(coreSolrServer, type); throw new PersistenceException( "RuntimeException while adding Solr index for persistent type " + type, e); } }
protected void execute() throws Exception { SolrServer server = new HttpSolrServer("http://localhost:8983/solr"); SolrQuery query = new SolrQuery("text:ipod"); QueryResponse response = server.query(query); SolrDocumentList docList = response.getResults(); System.out.println("件数:" + docList.size()); for (SolrDocument doc : docList) { System.out.println("■"); for (String fieldName : doc.getFieldNames()) { System.out.println(fieldName.toString() + ":" + doc.getFieldValue(fieldName.toString())); } } }