@Test
public void fieldsRequestsShouldTurnIntoSelectors() throws Exception, TException {
String table = "fieldsRequestsShouldTurnIntoSelectors";
SolrServer server =
TestTableCreator.newTable(table)
.withRowCount(1)
.withRecordsPerRow(2)
.withRecordColumns("fam.value", "fam.mvf")
.create();
SolrQuery query = new SolrQuery("value0-0");
query.setFields("fam.value");
QueryResponse response = server.query(query);
assertEquals(
"We should get our doc back for a valid test.", 1l, response.getResults().getNumFound());
SolrDocument docResult = response.getResults().get(0);
assertEquals("value0-0", docResult.getFieldValue("fam.value"));
assertNull(
"We shouldn't get this one back since it wasnt in our fields.",
docResult.getFieldValues("fam.mvf"));
removeTable(table);
}
@RequestMapping("/search/{categoryId}")
public String list(@PathVariable String categoryId, ModelMap model)
throws SolrServerException, ParseException {
SolrQuery query = new SolrQuery();
query.setQuery("categoryId:" + categoryId);
query.setRows(20);
SolrDocumentList docs = solrServer.query(query).getResults();
List<Article> articles = new ArrayList<Article>();
for (SolrDocument solrDocument : docs) {
Article article = new Article();
article.setTitle(String.valueOf(solrDocument.getFieldValue("title")));
article.setContent(String.valueOf(solrDocument.getFieldValue("content")));
/** solr时间格式转换的坑,具体可参考 http://chengqianl.iteye.com/blog/1340385 */
article.setCreateDate(
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.parse(String.valueOf(solrDocument.getFieldValue("createDate"))));
articles.add(article);
}
model.addAttribute("articles", articles);
return "/client/article/list";
}
private static SolrDocument toSolrDocument(SolrInputDocument d) {
SolrDocument doc = new SolrDocument();
for (String name : d.getFieldNames()) {
doc.addField(name, d.getFieldValue(name));
}
return doc;
}
public static void main(String a[]) throws MalformedURLException, SolrServerException {
SolrServer hotServer = new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds");
SolrQuery solrLocrQuery = new SolrQuery();
/*solrLocrQuery.setParam("pos", "ORB");
solrLocrQuery.setParam("locale", "en_US");*/
solrLocrQuery.setRows(300);
QueryResponse queryLocResponse = hotServer.query(solrLocrQuery);
SolrDocumentList locList = queryLocResponse.getResults();
for (SolrDocument doc : locList) {
try {
String str = doc.get("text_s").toString();
if (str.toLowerCase().contains("price")) {
if (str.toLowerCase().contains("assurance")) {
String id = doc.get("id").toString();
String url = doc.get("title_s").toString();
System.out.println(id + "|" + url);
}
}
} catch (Exception e) {
}
}
}
/**
* @param d SolrInputDocument to convert
* @return a SolrDocument with the same fields and values as the SolrInputDocument
*/
public static SolrDocument toSolrDocument(SolrInputDocument d) {
SolrDocument doc = new SolrDocument();
for (SolrInputField field : d) {
doc.setField(field.getName(), field.getValue());
}
return doc;
}
@Override
public Pair<DocumentReference, String> next() {
SolrDocument result = getResults().get(index++);
DocumentReference documentReference = this.solrDocumentReferenceResolver.resolve(result);
String version = (String) result.get(FieldUtils.VERSION);
return new ImmutablePair<DocumentReference, String>(documentReference, version);
}
public SearchResponse search(SearchRequest request) {
SolrQuery query = new SolrQuery();
query.setRows(request.getLimit());
query.setStart(request.getOffset() * request.getLimit());
BooleanQuery aggregate = new BooleanQuery();
if (!StringUtils.isBlank(request.getText())) {
TermQuery termQuery = new TermQuery(new Term("", request.getText()));
aggregate.add(termQuery, BooleanClause.Occur.SHOULD);
}
if (!StringUtils.isBlank(request.getEventId())) {
aggregate.add(
new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST);
}
query.setQuery(aggregate.toString());
log.info("QUERY IS: " + query.toString());
try {
QueryResponse queryResponse = solrServer.query(query);
log.info("RESPONSE FROM QUERY WAS: " + queryResponse);
SolrDocumentList results = queryResponse.getResults();
log.info("RESULTS WAS: " + results);
ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>();
for (SolrDocument result : results) {
hits.add(
new SearchResponse.Hit(
ObjectType.session, String.valueOf(result.getFieldValue("id")), null));
}
return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits);
} catch (SolrServerException e) {
e.printStackTrace();
}
return null;
}
@Test
public void testFieldMutating() throws Exception {
HttpSolrClient client = (HttpSolrClient) getSolrClient();
client.deleteByQuery("*:*");
client.commit();
assertNumFound("*:*", 0); // make sure it got in
// two docs, one with uniqueKey, another without it
String json =
"{\"name one\": \"name\"} "
+ "{\"name two\" : \"name\"}"
+ "{\"first-second\" : \"name\"}"
+ "{\"x+y\" : \"name\"}"
+ "{\"p%q\" : \"name\"}"
+ "{\"p.q\" : \"name\"}"
+ "{\"a&b\" : \"name\"}";
HttpClient httpClient = client.getHttpClient();
HttpPost post = new HttpPost(client.getBaseURL() + "/update/json/docs");
post.setHeader("Content-Type", "application/json");
post.setEntity(new InputStreamEntity(new ByteArrayInputStream(json.getBytes("UTF-8")), -1));
HttpResponse response = httpClient.execute(post);
assertEquals(200, response.getStatusLine().getStatusCode());
client.commit();
List<String> expected =
Arrays.asList("name_one", "name__two", "first-second", "a_b", "p_q", "p.q", "x_y");
HashSet set = new HashSet();
QueryResponse rsp = assertNumFound("*:*", expected.size());
for (SolrDocument doc : rsp.getResults()) set.addAll(doc.getFieldNames());
for (String s : expected) {
assertTrue(s + " not created " + rsp, set.contains(s));
}
}
public void writeSolrDocument(SolrDocument doc) throws IOException {
List<SolrDocument> children = doc.getChildDocuments();
int fieldsCount = 0;
if (writableDocFields == null || writableDocFields.wantsAllFields() || ignoreWritable) {
fieldsCount = doc.size();
} else {
for (Entry<String, Object> e : doc) {
if (toWrite(e.getKey())) fieldsCount++;
}
}
int sz = fieldsCount + (children == null ? 0 : children.size());
writeTag(SOLRDOC);
writeTag(ORDERED_MAP, sz);
for (Map.Entry<String, Object> entry : doc) {
String name = entry.getKey();
if (toWrite(name)) {
writeExternString(name);
Object val = entry.getValue();
writeVal(val);
}
}
if (children != null) {
try {
ignoreWritable = true;
for (SolrDocument child : children) {
writeSolrDocument(child);
}
} finally {
ignoreWritable = false;
}
}
}
@Test
public void basicFullTextQuery() throws Exception {
String table = "basicFullTextQuery";
SolrServer server =
TestTableCreator.newTable(table)
.withRowCount(1)
.withRecordsPerRow(2)
.withRecordColumns("fam.value", "fam.mvf", "fam.mvf")
.create();
SolrQuery query = new SolrQuery("value0-0");
QueryResponse response = server.query(query);
assertEquals("We should get our doc back.", 1l, response.getResults().getNumFound());
SolrDocument docResult = response.getResults().get(0);
assertEquals("0", docResult.getFieldValue("recordid"));
assertEquals("value0-0", docResult.getFieldValue("fam.value"));
Collection<Object> mvfVals = docResult.getFieldValues("fam.mvf");
assertTrue(
"We should get all our values back[" + mvfVals + "]",
CollectionUtils.isEqualCollection(mvfVals, Lists.newArrayList("value0-0", "value0-0")));
removeTable(table);
}
private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
// Keep in mind that this could also be a shard in a multi-tiered system.
// TODO: if a multi-tiered system, it seems like some requests
// could/should bypass middlemen (like retrieving stored fields)
// TODO: merge fsv to if requested
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
assert (sreq.responses.size() == 1);
ShardResponse srsp = sreq.responses.get(0);
SolrDocumentList docs =
(SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName();
for (SolrDocument doc : docs) {
Object id = doc.getFieldValue(keyFieldName);
ShardDoc sdoc = rb.resultIds.get(id.toString());
if (sdoc != null) {
if (returnScores && sdoc.score != null) {
doc.setField("score", sdoc.score);
}
rb._responseDocs.set(sdoc.positionInResponse, doc);
}
}
}
}
private void assertScore(boolean isScoresTest, SolrDocument doc) {
if (isScoresTest) {
assertThat(
"score join doesn't return 1.0", doc.getFirstValue("score").toString(), not("1.0"));
} else {
assertEquals("Solr join has constant score", "1.0", doc.getFirstValue("score").toString());
}
}
public SolrSearchDocument(String id, String resourceId, String context) {
this();
doc.put(SearchFields.ID_FIELD_NAME, id);
doc.put(SearchFields.URI_FIELD_NAME, resourceId);
if (context != null) {
doc.put(SearchFields.CONTEXT_FIELD_NAME, context);
}
}
public static void removeOtherFields(SolrDocument d, Set<String> fields) {
Collection<String> fieldNames = new HashSet<String>(d.getFieldNames());
for (String fieldName : fieldNames) {
// if field is not listed among fields, then delete it
if ((!fields.contains(fieldName)) && (!fieldName.equals("score"))) {
d.removeFields(fieldName);
}
}
}
public JSONArray Search(String query, String sort) {
SolrServer server;
SolrDocumentList docs = null;
String Date_format = null; // for Date格式轉換
JSONArray jsonArray = new JSONArray();
try {
server = new HttpSolrServer(url);
// add
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery(query);
solrQuery.setStart(_start);
solrQuery.setRows(_nbDocuments);
solrQuery.setRequestHandler("query");
solrQuery.set("fl", "*,score"); // 設定fl參數，指定要傳回哪個field的資料，這裡設定所有field與score
if (_fq1 != "") {
solrQuery.addFilterQuery("ProductName:" + _fq1);
}
if (_fq2 != "") {
solrQuery.addFilterQuery("publishedDate:" + _fq2);
}
if (sort != null) {
solrQuery.addSortField(sort, ORDER.asc);
}
solrQuery.setRequestHandler("/browse");
response = server.query(solrQuery);
docs = response.getResults();
if (docs != null) {
System.out.println(
docs.getNumFound() + " documents found, " + docs.size() + " returned : ");
setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
JSONObject jsonObject = new JSONObject();
for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) {
Entry<String, Object> entry = it2.next();
if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串，方便前端呈現
Date_format = entry.getValue().toString();
jsonObject.put(entry.getKey(), Date_format);
} else {
// 一般情況
jsonObject.put(entry.getKey(), entry.getValue());
}
}
System.out.print("\n");
// 將總共找到幾筆資料存在jsonarray的最後面傳給前端
jsonObject.put("TotalResultFound", docs.getNumFound());
jsonArray.add(jsonObject);
}
}
} catch (SolrServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return jsonArray;
}
protected FL_Link buildResultFromDocument(SolrDocument sd) {
FL_Link.Builder linkBuilder = FL_Link.newBuilder();
FL_PropertyDescriptors linkDescriptors = _applicationConfiguration.getLinkDescriptors();
FL_PropertyDescriptors entityDescriptors = _applicationConfiguration.getEntityDescriptors();
final String type = getTypeFromDocument(sd, linkDescriptors);
Boolean isMultitype = (entityDescriptors.getTypes().size() > 1);
String uid =
sd.getFieldValue(
PropertyDescriptorHelper.mapKey(
FL_RequiredPropertyKey.ID.name(), linkDescriptors.getProperties(), type))
.toString();
Object oSource =
sd.getFieldValue(
PropertyDescriptorHelper.mapKey(
FL_RequiredPropertyKey.FROM.name(), linkDescriptors.getProperties(), type));
String sSource =
oSource instanceof String
? (String) oSource
: oSource instanceof Integer ? Integer.toString((Integer) oSource) : null;
linkBuilder.setProvenance(null);
linkBuilder.setUncertainty(null);
linkBuilder.setType(type);
List<FL_Property> props = getPropertiesFromDocument(sd, type, linkDescriptors.getProperties());
linkBuilder.setProperties(props);
linkBuilder.setLinkTypes(null);
linkBuilder.setUid(_namespaceHandler.globalFromLocalEntityId(InfluentId.LINK, type, uid));
String sourceUID = null;
if (sSource != null) {
InfluentId infId;
if (isMultitype) {
infId = InfluentId.fromTypedId(InfluentId.ACCOUNT, sSource);
} else {
infId =
InfluentId.fromNativeId(
InfluentId.ACCOUNT, entityDescriptors.getTypes().get(0).getKey(), sSource);
}
sourceUID = infId.toString();
}
linkBuilder.setSource(sourceUID);
linkBuilder.setTarget(getTarget(sd));
return linkBuilder.build();
}
@Override
public T convert(Map<String, ?> source) {
if (source == null) {
return null;
}
SolrDocument document = new SolrDocument();
document.putAll(source);
return documentObjectBinder.getBean(clazz, document);
}
private void assertOrder(QueryResponse rsp, String... docs) throws Exception {
SolrDocumentList list = rsp.getResults();
for (int i = 0; i < docs.length; i++) {
SolrDocument doc = list.get(i);
Object o = doc.getFieldValue("id");
if (!docs[i].equals(o)) {
throw new Exception("Order is not correct:" + o + "!=" + docs[i]);
}
}
}
public void testSolrDocumentEquals() {
String randomString = TestUtil.randomSimpleString(random());
SolrDocument doc1 = new SolrDocument();
doc1.addField("foo", randomString);
SolrDocument doc2 = new SolrDocument();
doc2.addField("foo", randomString);
assertTrue(assertSolrDocumentEquals(doc1, doc2));
doc1.addField("foo", "bar");
assertFalse(assertSolrDocumentEquals(doc1, doc2));
doc1 = new SolrDocument();
doc1.addField("bar", randomString);
assertFalse(assertSolrDocumentEquals(doc1, doc2));
int randomInt = random().nextInt();
doc1 = new SolrDocument();
doc1.addField("foo", randomInt);
doc2 = new SolrDocument();
doc2.addField("foo", randomInt);
assertTrue(assertSolrDocumentEquals(doc1, doc2));
doc2 = new SolrDocument();
doc2.addField("bar", randomInt);
assertFalse(assertSolrDocumentEquals(doc1, doc2));
}
private static Set<String> getStringSet(SolrDocument doc, SolrName field) {
Set<String> set = new HashSet<String>();
if (doc.containsKey(field.solrName())) {
for (Object val : doc.getFieldValues(field.solrName())) {
set.add((String) val);
}
}
return set;
}
@RequestMapping("/search")
public String search(ModelMap model) throws SolrServerException {
SolrQuery query = new SolrQuery();
/** 设置多条filterQuery语句有助于提高内存效率；组合查询条件设置单条filterQuery语句损耗内存多，但可以优化性能 可以采用 AND 或 OR 关键字 来设置查询与或 */
query.setQuery("title:北京 OR content:北京");
query.setRows(20);
// 设置高亮
query.setHighlight(true); // 开启高亮组件
query.addHighlightField("title"); // 高亮字段
query.addHighlightField("content"); // 高亮字段
query.setHighlightSimplePre("<font color='red'>"); // 标记，高亮关键字前缀
query.setHighlightSimplePost("</font>"); // 后缀
query
.setHighlight(true)
.setHighlightSnippets(
1); // 获取高亮分片数，一般搜索词可能分布在文章中的不同位置，其所在一定长度的语句即为一个片段，默认为1，但根据业务需要有时候需要多取出几个分片。 -
// 此处设置决定下文中titleList, contentList中元素的个数
query.setHighlightFragsize(150); // 每个分片的最大长度，默认为100。适当设置此值，如果太小，高亮的标题可能会显不全；设置太大，摘要可能会太长。
QueryResponse response = solrServer.query(query);
SolrDocumentList docs = response.getResults();
// 获取所有高亮的字段
Map<String, Map<String, List<String>>> highlightMap = response.getHighlighting();
List<Article> articles = new ArrayList<Article>();
for (SolrDocument solrDocument : docs) {
Article article = new Article();
List<String> titleList = highlightMap.get(solrDocument.getFieldValue("id")).get("title");
List<String> contentList = highlightMap.get(solrDocument.getFieldValue("id")).get("content");
// 获取并设置高亮的字段title
if (titleList != null && titleList.size() > 0) {
article.setTitle(titleList.get(0));
}
// 获取并设置高亮的字段content
if (contentList != null && contentList.size() > 0) {
article.setContent(contentList.get(0));
}
articles.add(article);
}
model.addAttribute("articles", articles);
return "/client/article/list";
}
public static boolean isLocalDirectory(SolrDocument doc) {
if (doc.containsKey(TITLE)) {
Object title = doc.get(TITLE);
String titleContents = "";
if (title instanceof ArrayList) {
titleContents = (String) ((ArrayList) title).get(0);
} else if (title instanceof String) {
titleContents = (String) title;
}
return titleContents.startsWith("Index of /");
}
return false;
}
/** Prepend the server context path to the location of search result documents. */
private void setSearchResultContext(NamedList<Object> values, String contextPath) {
// find the search result documents in the search response
SolrDocumentList documents = (SolrDocumentList) values.get("response"); // $NON-NLS-1$
if (documents == null) return;
for (SolrDocument doc : documents) {
String location = (String) doc.getFieldValue(ProtocolConstants.KEY_LOCATION);
if (location != null) {
// prepend the context path and update the document
location = contextPath + location;
doc.setField(ProtocolConstants.KEY_LOCATION, location);
}
}
}
public SolrRecord(SolrDocument doc, List<String> fields) {
JSONObject jsonObject = new JSONObject();
for (String s : fields) {
if (doc.containsKey(s)) {
jsonObject.put(s, doc.get(s));
}
}
String contentStr = jsonObject.toString(3).replaceAll("\n", DEFAULT_NEWLINE);
content = contentStr.getBytes();
contentType = SolrUtils.getDocumentContentType(doc);
id = getFieldStringValue(doc, ID, "not_found");
fileName = new File(id).getName();
}
public StringBuffer construct_lucene_solr(StringBuffer result, SolrDocument d) {
for (String field : (d.getFieldNames())) {
if ((!field.endsWith("_str")) && (!field.equalsIgnoreCase("fullrecord"))) {
Iterator j = d.getFieldValues(field).iterator();
while (j.hasNext()) {
result.append("<" + field + ">");
result.append(helpers.xmlEncode((String) j.next().toString()));
field = field.split(" ")[0];
result.append("</" + field + ">");
}
}
}
return (result);
}
private void writeSolr(SpectralClustering cluster) {
Map<Integer, List<Doc>> map = new HashMap<Integer, List<Doc>>();
int[] lab = cluster.getClusterLabel();
for (int x = 0; x < lab.length; x++) {
final Doc doc = list.get(x);
if (!map.containsKey(new Integer(lab[x]))) {
map.put(
new Integer(lab[x]),
new ArrayList<Doc>() {
{
add(doc);
}
});
} else {
map.get(new Integer(lab[x])).add(doc);
}
}
for (Map.Entry<Integer, List<Doc>> e : map.entrySet()) {
System.out.println("type:" + e.getKey());
if (e.getValue().size() < 10) {
StringBuffer query = new StringBuffer();
for (Doc doc : e.getValue()) {
query.append("url:\"").append(doc.url).append("\"");
query.append(" OR ");
}
query.delete(query.lastIndexOf("OR"), query.length());
String groupId = UUID.nameUUIDFromBytes(query.toString().getBytes()).toString();
List<Map<String, Object>> toSave = new ArrayList<Map<String, Object>>();
System.out.println("group:" + groupId + " = " + e.getValue().size());
List<SolrDocument> list =
(List<SolrDocument>) indexDao.sortList(query.toString(), 1, 100, "infoTime_dt desc");
Date newest = new Date(0);
int useful = 1;
for (SolrDocument doc : list) {
Date date = (Date) doc.get("infoTime_dt");
if (date.getTime() > newest.getTime()) {}
Map<String, Object> inputDoc = new HashMap<String, Object>(doc);
inputDoc.put("useful_i", useful);
inputDoc.put("sim_i", list.size());
inputDoc.put("group_s", groupId);
toSave.add(inputDoc);
useful = 0;
}
indexDao.addIndex(toSave);
}
}
}
protected void execute() throws Exception {
SolrServer server = new HttpSolrServer("http://localhost:8983/solr");
SolrQuery query = new SolrQuery("text:ipod");
QueryResponse response = server.query(query);
SolrDocumentList docList = response.getResults();
System.out.println("件数:" + docList.size());
for (SolrDocument doc : docList) {
System.out.println("■");
for (String fieldName : doc.getFieldNames()) {
System.out.println(fieldName.toString() + ":" + doc.getFieldValue(fieldName.toString()));
}
}
}
@SuppressWarnings("unused")
private void printHits(SolrQuery q) throws SolrServerException {
System.out.println("--- solr contents " + q.toString() + " ---");
SolrDocumentList results = getSolr().query(q).getResults();
if (results.size() == 0) {
System.out.println("empty");
}
for (SolrDocument d : results) {
for (String f : d.getFieldNames()) {
String v = "" + d.get(f);
System.out.print(", " + f + ": " + v);
}
System.out.println("");
}
}
// Build an FL_Link result from a list of SolrDocuments returned from a group command in a query
protected FL_Link buildResultFromGroupedDocuments(SolrDocumentList dl) {
// Build the initial result from the first document
FL_Link link = buildResultFromDocument(dl.get(0));
// Get the nodetype
String targetField =
PropertyDescriptorHelper.mapKey(
FL_RequiredPropertyKey.TO.name(),
_applicationConfiguration.getLinkDescriptors().getProperties(),
link.getType());
// Add the remaining document properties to the entity
// Currently only the TO field is aggregated from grouping of one-to-many links
for (int i = 1; i < dl.size(); i++) {
SolrDocument sd = dl.get(i);
String target = (String) sd.getFieldValue(targetField);
FL_Property property = null;
List<FL_Property> properties = link.getProperties();
for (FL_Property prop : link.getProperties()) {
if (prop.getKey().equals(FL_RequiredPropertyKey.TO.name())) {
property = prop;
}
}
if (property != null) {
Object range = property.getRange();
if (range instanceof FL_ListRange) {
List<Object> values = ((FL_ListRange) range).getValues();
values.add(target);
} else if (range instanceof FL_SingletonRange) {
List<Object> values = new ArrayList<Object>();
values.add(((FL_SingletonRange) range).getValue());
values.add(target);
property.setRange(
FL_ListRange.newBuilder().setType(FL_PropertyType.STRING).setValues(values).build());
}
link.setProperties(properties);
}
link.setTarget(link.getTarget() + "," + getTarget(sd));
}
return link;
}
/**
* Returns a mods document, which is embedded in escaped xml form in the originalMods field of
* each solr document. This escaped string is unmarshalled (jaxb) into a loc.gov.mods.v3.ModsType
* (and all child elements) and returned as an individual item or added to a search result set
*
* @param doc a SolrDocument from which to extract the mods record
* @return the ModsType
* @see ModsType
*/
protected ModsType getModsType(SolrDocument doc) throws JAXBException {
String modsString = (String) doc.getFieldValue("originalMods");
Unmarshaller unmarshaller = JAXBHelper.context.createUnmarshaller();
StringReader reader = new StringReader(modsString);
ModsType modsType = (ModsType) ((JAXBElement) (unmarshaller.unmarshal(reader))).getValue();
return modsType;
}