public static void main(String a[]) throws MalformedURLException, SolrServerException {
SolrServer hotServer = new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds");
SolrQuery solrLocrQuery = new SolrQuery();
/*solrLocrQuery.setParam("pos", "ORB");
solrLocrQuery.setParam("locale", "en_US");*/
solrLocrQuery.setRows(300);
QueryResponse queryLocResponse = hotServer.query(solrLocrQuery);
SolrDocumentList locList = queryLocResponse.getResults();
for (SolrDocument doc : locList) {
try {
String str = doc.get("text_s").toString();
if (str.toLowerCase().contains("price")) {
if (str.toLowerCase().contains("assurance")) {
String id = doc.get("id").toString();
String url = doc.get("title_s").toString();
System.out.println(id + "|" + url);
}
}
} catch (Exception e) {
}
}
}
@Override
public Pair<DocumentReference, String> next() {
SolrDocument result = getResults().get(index++);
String wiki = (String) result.get(FieldUtils.WIKI);
String space = (String) result.get(FieldUtils.SPACE);
String name = (String) result.get(FieldUtils.NAME);
String locale = (String) result.get(FieldUtils.DOCUMENT_LOCALE);
String version = (String) result.get(FieldUtils.VERSION);
DocumentReference documentReference = new DocumentReference(wiki, space, name);
if (!locale.isEmpty()) {
documentReference = new DocumentReference(documentReference, LocaleUtils.toLocale(locale));
}
return new ImmutablePair<DocumentReference, String>(documentReference, version);
}
// 查询
@Test
public void testSolrForSearch() throws Exception {
SolrQuery params = new SolrQuery();
params.set("q", "*:*");
QueryResponse response = solrServer.query(params);
// 结果集
SolrDocumentList docs = response.getResults();
long count = docs.getNumFound();
System.out.println("结果集有 ：" + count);
for (SolrDocument doc : docs) {
String id = (String) doc.get("id");
String name = (String) doc.get("name");
System.out.println("id :" + id + "---" + "name :" + name);
}
}
@Override
public Pair<DocumentReference, String> next() {
SolrDocument result = getResults().get(index++);
DocumentReference documentReference = this.solrDocumentReferenceResolver.resolve(result);
String version = (String) result.get(FieldUtils.VERSION);
return new ImmutablePair<DocumentReference, String>(documentReference, version);
}
@Override
public List<Message> GetResult(String word) {
List<Message> messages = new ArrayList<Message>();
try {
sQuery.setQuery(word);
QueryResponse response = solr.query(sQuery);
SolrDocumentList reList = response.getResults();
for (SolrDocument s : reList) {
String id = (String) s.get("id");
String des = (String) s.get("description");
String s1 = s.get("title").toString();
messages.add(new Message(id, des, s1));
}
return messages;
} catch (SolrServerException | IOException e) {
e.printStackTrace();
return null;
}
}
/**
* Extract an {@link EntityReference} from the given {@link SolrDocument} (e.g. search result).
* The entity type is inferred from the "type" field which must be specified and must have a valid
* value (that corresponds to an existing {@link EntityType}).
*
* @param document a {@link SolrDocument} to extract the {@link EntityReference} from (the "type"
* field must be specified)
* @param parameters the parameters to pass to the reference resolver (e.g. in case some reference
* components are missing)
* @return the reference to the entity associated with the given {@link SolrDocument}
*/
public EntityReference resolve(SolrDocument document, Object... parameters) {
EntityType type;
try {
type = EntityType.valueOf((String) document.get(FieldUtils.TYPE));
} catch (IllegalArgumentException e) {
return null;
} catch (NullPointerException e) {
return null;
}
return resolve(document, type, parameters);
}
public static boolean isLocalDirectory(SolrDocument doc) {
if (doc.containsKey(TITLE)) {
Object title = doc.get(TITLE);
String titleContents = "";
if (title instanceof ArrayList) {
titleContents = (String) ((ArrayList) title).get(0);
} else if (title instanceof String) {
titleContents = (String) title;
}
return titleContents.startsWith("Index of /");
}
return false;
}
@Override
public boolean hasProperty(String name, String value) {
List<String> fieldValues = asStringList(doc.get(name));
if (fieldValues != null) {
for (String fieldValue : fieldValues) {
if (value.equals(fieldValue)) {
return true;
}
}
}
return false;
}
public void run() {
List<DBObject> conversations = conversationsRepository.getConversations();
for (DBObject conversation : conversations) {
List<SolrDocument> associatedTweets =
rootTweetSolrRepository.findAssociatedTweets(
valueOf(conversation.get("text")),
valueOf(conversation.get("url")),
valueOf(conversation.get("id")));
List<Map> associatedTweetsInfo = new ArrayList<>();
for (SolrDocument associatedTweet : associatedTweets) {
associatedTweetsInfo.add(
of(
"text",
associatedTweet.get("text"),
"id",
associatedTweet.get("id"),
"url",
associatedTweet.get("url") == null ? "" : associatedTweet.get("url")));
}
conversation.put("associated_tweets", associatedTweetsInfo);
}
conversationsRepository.update(conversations);
}
public SolrRecord(SolrDocument doc, List<String> fields) {
JSONObject jsonObject = new JSONObject();
for (String s : fields) {
if (doc.containsKey(s)) {
jsonObject.put(s, doc.get(s));
}
}
String contentStr = jsonObject.toString(3).replaceAll("\n", DEFAULT_NEWLINE);
content = contentStr.getBytes();
contentType = SolrUtils.getDocumentContentType(doc);
id = getFieldStringValue(doc, ID, "not_found");
fileName = new File(id).getName();
}
private void writeSolr(SpectralClustering cluster) {
Map<Integer, List<Doc>> map = new HashMap<Integer, List<Doc>>();
int[] lab = cluster.getClusterLabel();
for (int x = 0; x < lab.length; x++) {
final Doc doc = list.get(x);
if (!map.containsKey(new Integer(lab[x]))) {
map.put(
new Integer(lab[x]),
new ArrayList<Doc>() {
{
add(doc);
}
});
} else {
map.get(new Integer(lab[x])).add(doc);
}
}
for (Map.Entry<Integer, List<Doc>> e : map.entrySet()) {
System.out.println("type:" + e.getKey());
if (e.getValue().size() < 10) {
StringBuffer query = new StringBuffer();
for (Doc doc : e.getValue()) {
query.append("url:\"").append(doc.url).append("\"");
query.append(" OR ");
}
query.delete(query.lastIndexOf("OR"), query.length());
String groupId = UUID.nameUUIDFromBytes(query.toString().getBytes()).toString();
List<Map<String, Object>> toSave = new ArrayList<Map<String, Object>>();
System.out.println("group:" + groupId + " = " + e.getValue().size());
List<SolrDocument> list =
(List<SolrDocument>) indexDao.sortList(query.toString(), 1, 100, "infoTime_dt desc");
Date newest = new Date(0);
int useful = 1;
for (SolrDocument doc : list) {
Date date = (Date) doc.get("infoTime_dt");
if (date.getTime() > newest.getTime()) {}
Map<String, Object> inputDoc = new HashMap<String, Object>(doc);
inputDoc.put("useful_i", useful);
inputDoc.put("sim_i", list.size());
inputDoc.put("group_s", groupId);
toSave.add(inputDoc);
useful = 0;
}
indexDao.addIndex(toSave);
}
}
}
@SuppressWarnings("unused")
private void printHits(SolrQuery q) throws SolrServerException {
System.out.println("--- solr contents " + q.toString() + " ---");
SolrDocumentList results = getSolr().query(q).getResults();
if (results.size() == 0) {
System.out.println("empty");
}
for (SolrDocument d : results) {
for (String f : d.getFieldNames()) {
String v = "" + d.get(f);
System.out.print(", " + f + ": " + v);
}
System.out.println("");
}
}
public boolean isSame(SolrIndexSearchMailDocument o) {
if (this == o) {
return true; // objects are the same
}
if (null == o) {
return false; // dest is null
}
if (m_doc != null && o.m_doc != null) {
if (m_doc.size() != o.m_doc.size()) {
return false; // docs have different number of elements
}
for (String fieldName : m_doc.getFieldNames()) {
if (!o.m_doc.containsKey(fieldName)) {
return false; // a key was found in src but not in the dest
}
final Object srcValue = m_doc.get(fieldName);
final Object destValue = o.m_doc.get(fieldName);
if (srcValue == null ^ destValue == null) {
return false; // one value is null
} else if (srcValue != null && destValue != null) {
if (srcValue instanceof Collection && destValue instanceof Collection) {
Collection srcThings = (Collection) srcValue;
Collection destThings = (Collection) destValue;
if (srcThings.size() != destThings.size()) {
return false; // src and dest contain different counts of elements
}
if (!srcThings.containsAll(destThings)) {
return false; // contents of src and dest do not match
}
} else if (!srcValue.equals(destValue)) {
return false; // values are different
}
} // else { both null and equal }
}
} else if ((m_doc != null) ^ (o.m_doc != null)) {
return false; // one has a document and the other doesn't
}
// else { both are null and equal }
return true;
}
public static final SolrDocument toSolrDocument(Document doc, final IndexSchema schema) {
SolrDocument out = new SolrDocument();
for (IndexableField f : doc.getFields()) {
// Make sure multivalued fields are represented as lists
Object existing = out.get(f.name());
if (existing == null) {
SchemaField sf = schema.getFieldOrNull(f.name());
if (sf != null && sf.multiValued()) {
List<Object> vals = new ArrayList<>();
vals.add(f);
out.setField(f.name(), vals);
} else {
out.setField(f.name(), f);
}
} else {
out.addField(f.name(), f);
}
}
return out;
}
@Test
public void testGetId() throws Exception {
SolrDocument rsp = getSolrClient().getById("0");
assertNull(rsp);
rsp = getSolrClient().getById("1");
assertEquals("1", rsp.get("id"));
assertEquals("Microsoft", rsp.get("term_s"));
assertEquals("MSFT", rsp.get("term2_s"));
rsp = getSolrClient().getById("2");
assertEquals("2", rsp.get("id"));
assertEquals("Apple", rsp.get("term_s"));
assertEquals("AAPL", rsp.get("term2_s"));
}
@Test
public void testGetIdWithParams() throws Exception {
final SolrParams ID_FL_ONLY = params(CommonParams.FL, "id");
SolrDocument rsp = getSolrClient().getById("0", ID_FL_ONLY);
assertNull(rsp);
rsp = getSolrClient().getById("1", ID_FL_ONLY);
assertEquals("1", rsp.get("id"));
assertNull("This field should have been removed from the response.", rsp.get("term_s"));
assertNull("This field should have been removed from the response.", rsp.get("term2_s"));
rsp = getSolrClient().getById("2", ID_FL_ONLY);
assertEquals("2", rsp.get("id"));
assertNull("This field should have been removed from the response.", rsp.get("term_s"));
assertNull("This field should have been removed from the response.", rsp.get("term2_s"));
}
private static SolrDocument toSolrDoc(StoredDocument doc, IndexSchema schema) {
SolrDocument out = new SolrDocument();
for (StorableField f : doc.getFields()) {
// Make sure multivalued fields are represented as lists
Object existing = out.get(f.name());
if (existing == null) {
SchemaField sf = schema.getFieldOrNull(f.name());
// don't return copyField targets
if (sf != null && schema.isCopyFieldTarget(sf)) continue;
if (sf != null && sf.multiValued()) {
List<Object> vals = new ArrayList<Object>();
vals.add(f);
out.setField(f.name(), vals);
} else {
out.setField(f.name(), f);
}
} else {
out.addField(f.name(), f);
}
}
return out;
}
private JSONObject runSearch(HashMap<String, JSONObject> currMap) throws ServletException {
JSONObject qJson = new JSONObject();
try {
for (Map.Entry<String, JSONObject> entry : currMap.entrySet()) {
JSONObject resultAllJson = new JSONObject();
String key = entry.getKey();
JSONObject json = (JSONObject) entry.getValue();
String queryVal = json.getString("query");
// System.out.println("query: " + json.toString());
// continue with properties list
String searchType = null;
int limit = 10; // default
String typeStrict = "should"; // default
ArrayList<String[]> propertiesList = new ArrayList<String[]>();
if (json.has("type")) {
searchType = json.getString("type");
}
if (json.has("limit")) {
limit = json.getInt("limit");
}
if (json.has("type_strict")) { // Not sure what this variable
// represents. Skip for now.
typeStrict = json.getString("type_strict");
}
if (json.has("properties")) {
JSONArray properties = json.getJSONArray("properties");
for (int i = 0; i < properties.length(); i++) {
String[] pvPair = new String[2];
JSONObject jsonProperty = properties.getJSONObject(i);
String pid = jsonProperty.getString("pid");
String v = jsonProperty.getString("v");
if (pid != null && v != null) {
pvPair[0] = pid;
pvPair[1] = v;
propertiesList.add(pvPair);
}
}
}
// begin search
JSONArray resultJsonArr = new JSONArray();
// Solr
SolrQuery query = getQuery(queryVal, searchType, limit, propertiesList);
QueryResponse queryResponse = null;
if (query != null) {
SolrServer solr = SolrSetup.getSolrServer(getServletContext());
queryResponse = solr.query(query);
} else {
log.error("Query for a search was null");
}
SolrDocumentList docs = null;
if (queryResponse != null) {
docs = queryResponse.getResults();
} else {
log.error("Query response for a search was null");
}
if (docs != null) {
List<SearchResult> results = new ArrayList<SearchResult>();
for (SolrDocument doc : docs) {
try {
String uri = doc.get(VitroSearchTermNames.URI).toString();
// RY 7/1/2011
// Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get()
// returns a list.
// Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get()
// could return a list
// But in fact: I'm no longer seeing any lists returned for individuals with multiple
// labels. Not sure
// if this is new behavior or what. ???
Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
String name = null;
if (nameRaw instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> nameRawList = (List<String>) nameRaw;
name = nameRawList.get(0);
} else {
name = (String) nameRaw;
}
SearchResult result = new SearchResult(name, uri);
// populate result for Google Refine
JSONObject resultJson = new JSONObject();
resultJson.put("score", doc.getFieldValue("score"));
String modUri = result.getUri().replace("#", "%23");
resultJson.put("id", modUri);
resultJson.put("name", result.getLabel());
Collection<Object> rdfTypes = doc.getFieldValues(VitroSearchTermNames.RDFTYPE);
JSONArray typesJsonArr = new JSONArray();
if (rdfTypes != null) {
for (Object rdfType : rdfTypes) {
// e.g.
// http://aims.fao.org/aos/geopolitical.owl#area
String type = (String) rdfType;
int lastIndex2 = type.lastIndexOf('/') + 1;
String typeName = type.substring(lastIndex2);
typeName = typeName.replace("#", ":");
JSONObject typesJson = new JSONObject();
typesJson.put("id", type);
typesJson.put("name", typeName);
typesJsonArr.put(typesJson);
}
}
resultJson.put("type", typesJsonArr);
resultJson.put("match", "false");
resultJsonArr.put(resultJson);
} catch (Exception e) {
log.error(
"problem getting usable individuals from search " + "hits" + e.getMessage());
}
}
} else {
log.error("Docs for a search was null");
}
resultAllJson.put("result", resultJsonArr);
qJson.put(key, resultAllJson);
// System.out.println("results: " + qJson.toString());
}
} catch (JSONException ex) {
log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet JSONException: " + ex);
} catch (SolrServerException ex) {
log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet SolrServerException: " + ex);
}
return qJson;
}
@Override
public List<String> getProperty(String name) {
return asStringList(doc.get(name));
}
@Override
public String getContext() {
return (String) doc.get(SearchFields.CONTEXT_FIELD_NAME);
}
@Override
public String getResource() {
return (String) doc.get(SearchFields.URI_FIELD_NAME);
}
@Override
public String getId() {
return (String) doc.get(SearchFields.ID_FIELD_NAME);
}
public static Object getFieldValue(SolrDocument doc, String fieldName) {
return (doc != null && doc.containsKey(fieldName)) ? doc.get(fieldName) : null;
}
@SuppressWarnings("unchecked")
public static void main(String a[]) throws Exception {
// PrintWriter writer = new PrintWriter( "/users/mkchakravarti/flights/cds_greg" );
SolrServer solrLocServer =
new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds");
// HashMap<String,String> locMap = getLocMap();
SolrQuery solrQuery = new SolrQuery();
solrQuery.addFilterQuery("status:publish", "contentType:TitleBodyMedialist");
solrQuery.setQueryType("standard");
for (int i = 0; i < 7; i++) {
PrintWriter writer = new PrintWriter("/users/mkchakravarti/flights/cds_greg_ttl_" + i);
solrQuery.setStart(i * 20000);
solrQuery.setRows(20000);
solrQuery.setFields(
"urlType",
"urlMapping",
"contentPurpose",
"contentType",
"namedAttribute",
"locale",
"pos",
"steerModuleKey");
QueryResponse queryLocResponse = solrLocServer.query(solrQuery);
System.out.println(solrQuery);
SolrDocumentList locList = queryLocResponse.getResults();
for (SolrDocument doc : locList) {
try {
String id = (String) doc.get("id");
String contentPurpose = (String) doc.get("contentPurpose");
ArrayList<String> steerModuleKey = (ArrayList<String>) doc.get("steerModuleKey");
String contentType = (String) doc.get("contentType");
ArrayList<String> obj = (ArrayList<String>) doc.get("namedAttribute");
ArrayList<String> ut = (ArrayList<String>) doc.get("urlType");
ArrayList<String> um = (ArrayList<String>) doc.get("urlMapping");
ArrayList<String> pos = (ArrayList<String>) doc.get("pos");
ArrayList<String> locale = (ArrayList<String>) doc.get("locale");
String finalStr =
id
+ "_"
+ contentPurpose
+ "_"
+ contentType
+ "_"
+ app(obj)
+ "_"
+ app(ut)
+ "_"
+ app(um)
+ "_"
+ app(pos)
+ "_"
+ app(locale)
+ "_"
+ app(steerModuleKey);
writer.println(finalStr);
} catch (Exception e) {
System.out.println(e);
}
}
}
}
