@Test public void fieldsRequestsShouldTurnIntoSelectors() throws Exception, TException { String table = "fieldsRequestsShouldTurnIntoSelectors"; SolrServer server = TestTableCreator.newTable(table) .withRowCount(1) .withRecordsPerRow(2) .withRecordColumns("fam.value", "fam.mvf") .create(); SolrQuery query = new SolrQuery("value0-0"); query.setFields("fam.value"); QueryResponse response = server.query(query); assertEquals( "We should get our doc back for a valid test.", 1l, response.getResults().getNumFound()); SolrDocument docResult = response.getResults().get(0); assertEquals("value0-0", docResult.getFieldValue("fam.value")); assertNull( "We shouldn't get this one back since it wasnt in our fields.", docResult.getFieldValues("fam.mvf")); removeTable(table); }
@RequestMapping("/search/{categoryId}") public String list(@PathVariable String categoryId, ModelMap model) throws SolrServerException, ParseException { SolrQuery query = new SolrQuery(); query.setQuery("categoryId:" + categoryId); query.setRows(20); SolrDocumentList docs = solrServer.query(query).getResults(); List<Article> articles = new ArrayList<Article>(); for (SolrDocument solrDocument : docs) { Article article = new Article(); article.setTitle(String.valueOf(solrDocument.getFieldValue("title"))); article.setContent(String.valueOf(solrDocument.getFieldValue("content"))); /** solr时间格式转换的坑,具体可参考 http://chengqianl.iteye.com/blog/1340385 */ article.setCreateDate( new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") .parse(String.valueOf(solrDocument.getFieldValue("createDate")))); articles.add(article); } model.addAttribute("articles", articles); return "/client/article/list"; }
private static SolrDocument toSolrDocument(SolrInputDocument d) { SolrDocument doc = new SolrDocument(); for (String name : d.getFieldNames()) { doc.addField(name, d.getFieldValue(name)); } return doc; }
public static void main(String a[]) throws MalformedURLException, SolrServerException { SolrServer hotServer = new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds"); SolrQuery solrLocrQuery = new SolrQuery(); /*solrLocrQuery.setParam("pos", "ORB"); solrLocrQuery.setParam("locale", "en_US");*/ solrLocrQuery.setRows(300); QueryResponse queryLocResponse = hotServer.query(solrLocrQuery); SolrDocumentList locList = queryLocResponse.getResults(); for (SolrDocument doc : locList) { try { String str = doc.get("text_s").toString(); if (str.toLowerCase().contains("price")) { if (str.toLowerCase().contains("assurance")) { String id = doc.get("id").toString(); String url = doc.get("title_s").toString(); System.out.println(id + "|" + url); } } } catch (Exception e) { } } }
/** * @param d SolrInputDocument to convert * @return a SolrDocument with the same fields and values as the SolrInputDocument */ public static SolrDocument toSolrDocument(SolrInputDocument d) { SolrDocument doc = new SolrDocument(); for (SolrInputField field : d) { doc.setField(field.getName(), field.getValue()); } return doc; }
@Override public Pair<DocumentReference, String> next() { SolrDocument result = getResults().get(index++); DocumentReference documentReference = this.solrDocumentReferenceResolver.resolve(result); String version = (String) result.get(FieldUtils.VERSION); return new ImmutablePair<DocumentReference, String>(documentReference, version); }
public SearchResponse search(SearchRequest request) { SolrQuery query = new SolrQuery(); query.setRows(request.getLimit()); query.setStart(request.getOffset() * request.getLimit()); BooleanQuery aggregate = new BooleanQuery(); if (!StringUtils.isBlank(request.getText())) { TermQuery termQuery = new TermQuery(new Term("", request.getText())); aggregate.add(termQuery, BooleanClause.Occur.SHOULD); } if (!StringUtils.isBlank(request.getEventId())) { aggregate.add( new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST); } query.setQuery(aggregate.toString()); log.info("QUERY IS: " + query.toString()); try { QueryResponse queryResponse = solrServer.query(query); log.info("RESPONSE FROM QUERY WAS: " + queryResponse); SolrDocumentList results = queryResponse.getResults(); log.info("RESULTS WAS: " + results); ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>(); for (SolrDocument result : results) { hits.add( new SearchResponse.Hit( ObjectType.session, String.valueOf(result.getFieldValue("id")), null)); } return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits); } catch (SolrServerException e) { e.printStackTrace(); } return null; }
@Test public void testFieldMutating() throws Exception { HttpSolrClient client = (HttpSolrClient) getSolrClient(); client.deleteByQuery("*:*"); client.commit(); assertNumFound("*:*", 0); // make sure it got in // two docs, one with uniqueKey, another without it String json = "{\"name one\": \"name\"} " + "{\"name two\" : \"name\"}" + "{\"first-second\" : \"name\"}" + "{\"x+y\" : \"name\"}" + "{\"p%q\" : \"name\"}" + "{\"p.q\" : \"name\"}" + "{\"a&b\" : \"name\"}"; HttpClient httpClient = client.getHttpClient(); HttpPost post = new HttpPost(client.getBaseURL() + "/update/json/docs"); post.setHeader("Content-Type", "application/json"); post.setEntity(new InputStreamEntity(new ByteArrayInputStream(json.getBytes("UTF-8")), -1)); HttpResponse response = httpClient.execute(post); assertEquals(200, response.getStatusLine().getStatusCode()); client.commit(); List<String> expected = Arrays.asList("name_one", "name__two", "first-second", "a_b", "p_q", "p.q", "x_y"); HashSet set = new HashSet(); QueryResponse rsp = assertNumFound("*:*", expected.size()); for (SolrDocument doc : rsp.getResults()) set.addAll(doc.getFieldNames()); for (String s : expected) { assertTrue(s + " not created " + rsp, set.contains(s)); } }
public void writeSolrDocument(SolrDocument doc) throws IOException { List<SolrDocument> children = doc.getChildDocuments(); int fieldsCount = 0; if (writableDocFields == null || writableDocFields.wantsAllFields() || ignoreWritable) { fieldsCount = doc.size(); } else { for (Entry<String, Object> e : doc) { if (toWrite(e.getKey())) fieldsCount++; } } int sz = fieldsCount + (children == null ? 0 : children.size()); writeTag(SOLRDOC); writeTag(ORDERED_MAP, sz); for (Map.Entry<String, Object> entry : doc) { String name = entry.getKey(); if (toWrite(name)) { writeExternString(name); Object val = entry.getValue(); writeVal(val); } } if (children != null) { try { ignoreWritable = true; for (SolrDocument child : children) { writeSolrDocument(child); } } finally { ignoreWritable = false; } } }
@Test public void basicFullTextQuery() throws Exception { String table = "basicFullTextQuery"; SolrServer server = TestTableCreator.newTable(table) .withRowCount(1) .withRecordsPerRow(2) .withRecordColumns("fam.value", "fam.mvf", "fam.mvf") .create(); SolrQuery query = new SolrQuery("value0-0"); QueryResponse response = server.query(query); assertEquals("We should get our doc back.", 1l, response.getResults().getNumFound()); SolrDocument docResult = response.getResults().get(0); assertEquals("0", docResult.getFieldValue("recordid")); assertEquals("value0-0", docResult.getFieldValue("fam.value")); Collection<Object> mvfVals = docResult.getFieldValues("fam.mvf"); assertTrue( "We should get all our values back[" + mvfVals + "]", CollectionUtils.isEqualCollection(mvfVals, Lists.newArrayList("value0-0", "value0-0"))); removeTable(table); }
private void returnFields(ResponseBuilder rb, ShardRequest sreq) { // Keep in mind that this could also be a shard in a multi-tiered system. // TODO: if a multi-tiered system, it seems like some requests // could/should bypass middlemen (like retrieving stored fields) // TODO: merge fsv to if requested if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) { boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0; assert (sreq.responses.size() == 1); ShardResponse srsp = sreq.responses.get(0); SolrDocumentList docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response"); String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName(); for (SolrDocument doc : docs) { Object id = doc.getFieldValue(keyFieldName); ShardDoc sdoc = rb.resultIds.get(id.toString()); if (sdoc != null) { if (returnScores && sdoc.score != null) { doc.setField("score", sdoc.score); } rb._responseDocs.set(sdoc.positionInResponse, doc); } } } }
private void assertScore(boolean isScoresTest, SolrDocument doc) { if (isScoresTest) { assertThat( "score join doesn't return 1.0", doc.getFirstValue("score").toString(), not("1.0")); } else { assertEquals("Solr join has constant score", "1.0", doc.getFirstValue("score").toString()); } }
public SolrSearchDocument(String id, String resourceId, String context) { this(); doc.put(SearchFields.ID_FIELD_NAME, id); doc.put(SearchFields.URI_FIELD_NAME, resourceId); if (context != null) { doc.put(SearchFields.CONTEXT_FIELD_NAME, context); } }
public static void removeOtherFields(SolrDocument d, Set<String> fields) { Collection<String> fieldNames = new HashSet<String>(d.getFieldNames()); for (String fieldName : fieldNames) { // if field is not listed among fields, then delete it if ((!fields.contains(fieldName)) && (!fieldName.equals("score"))) { d.removeFields(fieldName); } } }
public JSONArray Search(String query, String sort) { SolrServer server; SolrDocumentList docs = null; String Date_format = null; // for Date格式轉換 JSONArray jsonArray = new JSONArray(); try { server = new HttpSolrServer(url); // add SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(query); solrQuery.setStart(_start); solrQuery.setRows(_nbDocuments); solrQuery.setRequestHandler("query"); solrQuery.set("fl", "*,score"); // 設定fl參數,指定要傳回哪個field的資料,這裡設定所有field與score if (_fq1 != "") { solrQuery.addFilterQuery("ProductName:" + _fq1); } if (_fq2 != "") { solrQuery.addFilterQuery("publishedDate:" + _fq2); } if (sort != null) { solrQuery.addSortField(sort, ORDER.asc); } solrQuery.setRequestHandler("/browse"); response = server.query(solrQuery); docs = response.getResults(); if (docs != null) { System.out.println( docs.getNumFound() + " documents found, " + docs.size() + " returned : "); setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端 for (int i = 0; i < docs.size(); i++) { SolrDocument doc = docs.get(i); JSONObject jsonObject = new JSONObject(); for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) { Entry<String, Object> entry = it2.next(); if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串,方便前端呈現 Date_format = entry.getValue().toString(); jsonObject.put(entry.getKey(), Date_format); } else { // 一般情況 jsonObject.put(entry.getKey(), entry.getValue()); } } System.out.print("\n"); // 將總共找到幾筆資料存在jsonarray的最後面傳給前端 jsonObject.put("TotalResultFound", docs.getNumFound()); jsonArray.add(jsonObject); } } } catch (SolrServerException e) { // TODO Auto-generated catch block e.printStackTrace(); } return jsonArray; }
protected FL_Link buildResultFromDocument(SolrDocument sd) { FL_Link.Builder linkBuilder = FL_Link.newBuilder(); FL_PropertyDescriptors linkDescriptors = _applicationConfiguration.getLinkDescriptors(); FL_PropertyDescriptors entityDescriptors = _applicationConfiguration.getEntityDescriptors(); final String type = getTypeFromDocument(sd, linkDescriptors); Boolean isMultitype = (entityDescriptors.getTypes().size() > 1); String uid = sd.getFieldValue( PropertyDescriptorHelper.mapKey( FL_RequiredPropertyKey.ID.name(), linkDescriptors.getProperties(), type)) .toString(); Object oSource = sd.getFieldValue( PropertyDescriptorHelper.mapKey( FL_RequiredPropertyKey.FROM.name(), linkDescriptors.getProperties(), type)); String sSource = oSource instanceof String ? (String) oSource : oSource instanceof Integer ? Integer.toString((Integer) oSource) : null; linkBuilder.setProvenance(null); linkBuilder.setUncertainty(null); linkBuilder.setType(type); List<FL_Property> props = getPropertiesFromDocument(sd, type, linkDescriptors.getProperties()); linkBuilder.setProperties(props); linkBuilder.setLinkTypes(null); linkBuilder.setUid(_namespaceHandler.globalFromLocalEntityId(InfluentId.LINK, type, uid)); String sourceUID = null; if (sSource != null) { InfluentId infId; if (isMultitype) { infId = InfluentId.fromTypedId(InfluentId.ACCOUNT, sSource); } else { infId = InfluentId.fromNativeId( InfluentId.ACCOUNT, entityDescriptors.getTypes().get(0).getKey(), sSource); } sourceUID = infId.toString(); } linkBuilder.setSource(sourceUID); linkBuilder.setTarget(getTarget(sd)); return linkBuilder.build(); }
@Override public T convert(Map<String, ?> source) { if (source == null) { return null; } SolrDocument document = new SolrDocument(); document.putAll(source); return documentObjectBinder.getBean(clazz, document); }
private void assertOrder(QueryResponse rsp, String... docs) throws Exception { SolrDocumentList list = rsp.getResults(); for (int i = 0; i < docs.length; i++) { SolrDocument doc = list.get(i); Object o = doc.getFieldValue("id"); if (!docs[i].equals(o)) { throw new Exception("Order is not correct:" + o + "!=" + docs[i]); } } }
public void testSolrDocumentEquals() { String randomString = TestUtil.randomSimpleString(random()); SolrDocument doc1 = new SolrDocument(); doc1.addField("foo", randomString); SolrDocument doc2 = new SolrDocument(); doc2.addField("foo", randomString); assertTrue(assertSolrDocumentEquals(doc1, doc2)); doc1.addField("foo", "bar"); assertFalse(assertSolrDocumentEquals(doc1, doc2)); doc1 = new SolrDocument(); doc1.addField("bar", randomString); assertFalse(assertSolrDocumentEquals(doc1, doc2)); int randomInt = random().nextInt(); doc1 = new SolrDocument(); doc1.addField("foo", randomInt); doc2 = new SolrDocument(); doc2.addField("foo", randomInt); assertTrue(assertSolrDocumentEquals(doc1, doc2)); doc2 = new SolrDocument(); doc2.addField("bar", randomInt); assertFalse(assertSolrDocumentEquals(doc1, doc2)); }
private static Set<String> getStringSet(SolrDocument doc, SolrName field) { Set<String> set = new HashSet<String>(); if (doc.containsKey(field.solrName())) { for (Object val : doc.getFieldValues(field.solrName())) { set.add((String) val); } } return set; }
@RequestMapping("/search") public String search(ModelMap model) throws SolrServerException { SolrQuery query = new SolrQuery(); /** 设置多条filterQuery语句有助于提高内存效率;组合查询条件设置单条filterQuery语句损耗内存多,但可以优化性能 可以采用 AND 或 OR 关键字 来设置查询与或 */ query.setQuery("title:北京 OR content:北京"); query.setRows(20); // 设置高亮 query.setHighlight(true); // 开启高亮组件 query.addHighlightField("title"); // 高亮字段 query.addHighlightField("content"); // 高亮字段 query.setHighlightSimplePre("<font color='red'>"); // 标记,高亮关键字前缀 query.setHighlightSimplePost("</font>"); // 后缀 query .setHighlight(true) .setHighlightSnippets( 1); // 获取高亮分片数,一般搜索词可能分布在文章中的不同位置,其所在一定长度的语句即为一个片段,默认为1,但根据业务需要有时候需要多取出几个分片。 - // 此处设置决定下文中titleList, contentList中元素的个数 query.setHighlightFragsize(150); // 每个分片的最大长度,默认为100。适当设置此值,如果太小,高亮的标题可能会显不全;设置太大,摘要可能会太长。 QueryResponse response = solrServer.query(query); SolrDocumentList docs = response.getResults(); // 获取所有高亮的字段 Map<String, Map<String, List<String>>> highlightMap = response.getHighlighting(); List<Article> articles = new ArrayList<Article>(); for (SolrDocument solrDocument : docs) { Article article = new Article(); List<String> titleList = highlightMap.get(solrDocument.getFieldValue("id")).get("title"); List<String> contentList = highlightMap.get(solrDocument.getFieldValue("id")).get("content"); // 获取并设置高亮的字段title if (titleList != null && titleList.size() > 0) { article.setTitle(titleList.get(0)); } // 获取并设置高亮的字段content if (contentList != null && contentList.size() > 0) { article.setContent(contentList.get(0)); } articles.add(article); } model.addAttribute("articles", articles); return "/client/article/list"; }
public static boolean isLocalDirectory(SolrDocument doc) { if (doc.containsKey(TITLE)) { Object title = doc.get(TITLE); String titleContents = ""; if (title instanceof ArrayList) { titleContents = (String) ((ArrayList) title).get(0); } else if (title instanceof String) { titleContents = (String) title; } return titleContents.startsWith("Index of /"); } return false; }
/** Prepend the server context path to the location of search result documents. */ private void setSearchResultContext(NamedList<Object> values, String contextPath) { // find the search result documents in the search response SolrDocumentList documents = (SolrDocumentList) values.get("response"); // $NON-NLS-1$ if (documents == null) return; for (SolrDocument doc : documents) { String location = (String) doc.getFieldValue(ProtocolConstants.KEY_LOCATION); if (location != null) { // prepend the context path and update the document location = contextPath + location; doc.setField(ProtocolConstants.KEY_LOCATION, location); } } }
public SolrRecord(SolrDocument doc, List<String> fields) { JSONObject jsonObject = new JSONObject(); for (String s : fields) { if (doc.containsKey(s)) { jsonObject.put(s, doc.get(s)); } } String contentStr = jsonObject.toString(3).replaceAll("\n", DEFAULT_NEWLINE); content = contentStr.getBytes(); contentType = SolrUtils.getDocumentContentType(doc); id = getFieldStringValue(doc, ID, "not_found"); fileName = new File(id).getName(); }
public StringBuffer construct_lucene_solr(StringBuffer result, SolrDocument d) { for (String field : (d.getFieldNames())) { if ((!field.endsWith("_str")) && (!field.equalsIgnoreCase("fullrecord"))) { Iterator j = d.getFieldValues(field).iterator(); while (j.hasNext()) { result.append("<" + field + ">"); result.append(helpers.xmlEncode((String) j.next().toString())); field = field.split(" ")[0]; result.append("</" + field + ">"); } } } return (result); }
private void writeSolr(SpectralClustering cluster) { Map<Integer, List<Doc>> map = new HashMap<Integer, List<Doc>>(); int[] lab = cluster.getClusterLabel(); for (int x = 0; x < lab.length; x++) { final Doc doc = list.get(x); if (!map.containsKey(new Integer(lab[x]))) { map.put( new Integer(lab[x]), new ArrayList<Doc>() { { add(doc); } }); } else { map.get(new Integer(lab[x])).add(doc); } } for (Map.Entry<Integer, List<Doc>> e : map.entrySet()) { System.out.println("type:" + e.getKey()); if (e.getValue().size() < 10) { StringBuffer query = new StringBuffer(); for (Doc doc : e.getValue()) { query.append("url:\"").append(doc.url).append("\""); query.append(" OR "); } query.delete(query.lastIndexOf("OR"), query.length()); String groupId = UUID.nameUUIDFromBytes(query.toString().getBytes()).toString(); List<Map<String, Object>> toSave = new ArrayList<Map<String, Object>>(); System.out.println("group:" + groupId + " = " + e.getValue().size()); List<SolrDocument> list = (List<SolrDocument>) indexDao.sortList(query.toString(), 1, 100, "infoTime_dt desc"); Date newest = new Date(0); int useful = 1; for (SolrDocument doc : list) { Date date = (Date) doc.get("infoTime_dt"); if (date.getTime() > newest.getTime()) {} Map<String, Object> inputDoc = new HashMap<String, Object>(doc); inputDoc.put("useful_i", useful); inputDoc.put("sim_i", list.size()); inputDoc.put("group_s", groupId); toSave.add(inputDoc); useful = 0; } indexDao.addIndex(toSave); } } }
protected void execute() throws Exception { SolrServer server = new HttpSolrServer("http://localhost:8983/solr"); SolrQuery query = new SolrQuery("text:ipod"); QueryResponse response = server.query(query); SolrDocumentList docList = response.getResults(); System.out.println("件数:" + docList.size()); for (SolrDocument doc : docList) { System.out.println("■"); for (String fieldName : doc.getFieldNames()) { System.out.println(fieldName.toString() + ":" + doc.getFieldValue(fieldName.toString())); } } }
@SuppressWarnings("unused") private void printHits(SolrQuery q) throws SolrServerException { System.out.println("--- solr contents " + q.toString() + " ---"); SolrDocumentList results = getSolr().query(q).getResults(); if (results.size() == 0) { System.out.println("empty"); } for (SolrDocument d : results) { for (String f : d.getFieldNames()) { String v = "" + d.get(f); System.out.print(", " + f + ": " + v); } System.out.println(""); } }
// Build an FL_Link result from a list of SolrDocuments returned from a group command in a query protected FL_Link buildResultFromGroupedDocuments(SolrDocumentList dl) { // Build the initial result from the first document FL_Link link = buildResultFromDocument(dl.get(0)); // Get the nodetype String targetField = PropertyDescriptorHelper.mapKey( FL_RequiredPropertyKey.TO.name(), _applicationConfiguration.getLinkDescriptors().getProperties(), link.getType()); // Add the remaining document properties to the entity // Currently only the TO field is aggregated from grouping of one-to-many links for (int i = 1; i < dl.size(); i++) { SolrDocument sd = dl.get(i); String target = (String) sd.getFieldValue(targetField); FL_Property property = null; List<FL_Property> properties = link.getProperties(); for (FL_Property prop : link.getProperties()) { if (prop.getKey().equals(FL_RequiredPropertyKey.TO.name())) { property = prop; } } if (property != null) { Object range = property.getRange(); if (range instanceof FL_ListRange) { List<Object> values = ((FL_ListRange) range).getValues(); values.add(target); } else if (range instanceof FL_SingletonRange) { List<Object> values = new ArrayList<Object>(); values.add(((FL_SingletonRange) range).getValue()); values.add(target); property.setRange( FL_ListRange.newBuilder().setType(FL_PropertyType.STRING).setValues(values).build()); } link.setProperties(properties); } link.setTarget(link.getTarget() + "," + getTarget(sd)); } return link; }
/** * Returns a mods document, which is embedded in escaped xml form in the originalMods field of * each solr document. This escaped string is unmarshalled (jaxb) into a loc.gov.mods.v3.ModsType * (and all child elements) and returned as an individual item or added to a search result set * * @param doc a SolrDocument from which to extract the mods record * @return the ModsType * @see ModsType */ protected ModsType getModsType(SolrDocument doc) throws JAXBException { String modsString = (String) doc.getFieldValue("originalMods"); Unmarshaller unmarshaller = JAXBHelper.context.createUnmarshaller(); StringReader reader = new StringReader(modsString); ModsType modsType = (ModsType) ((JAXBElement) (unmarshaller.unmarshal(reader))).getValue(); return modsType; }