@Override
public void run() {
Random random = LuceneTestCase.random();
int numSearches = 0;
while (true && !stop) {
numSearches++;
try {
// to come to the aid of their country.
cloudClient.query(new SolrQuery(QUERIES[random.nextInt(QUERIES.length)]));
} catch (Exception e) {
System.err.println("QUERY REQUEST FAILED:");
e.printStackTrace();
if (e instanceof SolrServerException) {
System.err.println("ROOT CAUSE:");
((SolrServerException) e).getRootCause().printStackTrace();
}
queryFails.incrementAndGet();
}
try {
Thread.sleep(random.nextInt(4000) + 300);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
log.info("num searches done:" + numSearches + " with " + queryFails + " fails");
}
private SearchEngineResponse<MediaItem> search(SolrQuery query) {
SearchEngineResponse<MediaItem> response = new SearchEngineResponse<MediaItem>();
QueryResponse rsp;
try {
rsp = server.query(query);
} catch (SolrServerException e) {
e.printStackTrace();
Logger.getRootLogger().info(e.getMessage());
return null;
}
List<SolrMediaItem> solrItems = rsp.getBeans(SolrMediaItem.class);
List<MediaItem> mediaItems = new ArrayList<MediaItem>();
for (SolrMediaItem solrMediaItem : solrItems) {
try {
MediaItem mediaItem = solrMediaItem.toMediaItem();
String id = mediaItem.getId();
mediaItem.setId(id);
mediaItems.add(mediaItem);
} catch (MalformedURLException ex) {
Logger.getRootLogger().error(ex.getMessage());
}
}
response.setResults(mediaItems);
return response;
}
public void run() {
// create new collections rapid fire
Map<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
int cnt = random().nextInt(TEST_NIGHTLY ? 13 : 3) + 1;
for (int i = 0; i < cnt; i++) {
String collectionName = "awholynewstresscollection_" + name + "_" + i;
int numShards = _TestUtil.nextInt(random(), 0, shardCount * 2) + 1;
int replicationFactor = _TestUtil.nextInt(random(), 0, 3) + 1;
int maxShardsPerNode =
(((numShards * 2 * replicationFactor)
/ getCommonCloudSolrServer()
.getZkStateReader()
.getClusterState()
.getLiveNodes()
.size()))
+ 1;
CloudSolrServer client = null;
try {
if (i == 0) {
client = createCloudClient(null);
} else if (i == 1) {
client = createCloudClient(collectionName);
}
createCollection(
collectionInfos,
collectionName,
numShards,
replicationFactor,
maxShardsPerNode,
client,
null,
"conf1");
// remove collection
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", collectionName);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
if (client == null) {
client = createCloudClient(null);
}
client.request(request);
} catch (SolrServerException e) {
e.printStackTrace();
throw new RuntimeException(e);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
} finally {
if (client != null) client.shutdown();
}
}
}
public SearchResponse search(SearchRequest request) {
SolrQuery query = new SolrQuery();
query.setRows(request.getLimit());
query.setStart(request.getOffset() * request.getLimit());
BooleanQuery aggregate = new BooleanQuery();
if (!StringUtils.isBlank(request.getText())) {
TermQuery termQuery = new TermQuery(new Term("", request.getText()));
aggregate.add(termQuery, BooleanClause.Occur.SHOULD);
}
if (!StringUtils.isBlank(request.getEventId())) {
aggregate.add(
new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST);
}
query.setQuery(aggregate.toString());
log.info("QUERY IS: " + query.toString());
try {
QueryResponse queryResponse = solrServer.query(query);
log.info("RESPONSE FROM QUERY WAS: " + queryResponse);
SolrDocumentList results = queryResponse.getResults();
log.info("RESULTS WAS: " + results);
ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>();
for (SolrDocument result : results) {
hits.add(
new SearchResponse.Hit(
ObjectType.session, String.valueOf(result.getFieldValue("id")), null));
}
return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits);
} catch (SolrServerException e) {
e.printStackTrace();
}
return null;
}
public static void addDoc(String docnum, String payloads) {
// 创建doc文档
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", docnum);
doc.addField("payloads", payloads);
System.out.println();
System.out.println("docnum: " + docnum);
System.out.println("payloads: " + payloads);
log.info("docnum: " + docnum + "\tpayloads:" + payloads + "\twordcount:" + (j + existWords));
try {
// 添加一个doc文档
UpdateResponse response = server.add(doc);
// commit后才保存到索引库
server.commit();
/* System.out.println("Query Time：" + response.getQTime());
System.out.println("Elapsed Time：" + response.getElapsedTime());
System.out.println("Status：" + response.getStatus());*/
} catch (SolrServerException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
productsToFilter = new ArrayList<String>();
String queryString = "*:*";
String sortCriteria = "user_count_purchased desc";
ModifiableSolrParams solrParams = new ModifiableSolrParams();
QueryResponse response = null;
RecommendResponse searchResponse = new RecommendResponse();
try {
String filterQueryString =
RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter);
if (query.getProductIds() != null && query.getProductIds().size() > 0) {
productsToFilter.addAll(query.getProductIds());
}
if (query.getUser() != null) {
if (alreadyBoughtProducts != null) {
productsToFilter.addAll(alreadyBoughtProducts);
}
}
if (productsToFilter != null && productsToFilter.size() > 0) {
if (filterQueryString.length() > 0) {
filterQueryString += " OR ";
}
filterQueryString +=
RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter);
}
solrParams.set("q", queryString);
solrParams.set("fq", filterQueryString);
solrParams.set("sort", sortCriteria);
solrParams.set("rows", maxReuslts);
response =
SolrServiceContainer.getInstance()
.getRecommendService()
.getSolrServer()
.query(solrParams);
// fill response object
List<String> extractedRecommendations =
RecommendationQueryUtils.extractRecommendationIds(
response.getBeans(CustomerAction.class));
searchResponse.setResultItems(extractedRecommendations);
searchResponse.setElapsedTime(response.getElapsedTime());
SolrDocumentList docResults = response.getResults();
searchResponse.setNumFound(docResults.getNumFound());
} catch (SolrServerException e) {
e.printStackTrace();
searchResponse.setNumFound(0);
searchResponse.setResultItems(new ArrayList<String>());
searchResponse.setElapsedTime(-1);
}
return searchResponse;
}
public JSONArray Search(String query, String sort) {
SolrServer server;
SolrDocumentList docs = null;
String Date_format = null; // for Date格式轉換
JSONArray jsonArray = new JSONArray();
try {
server = new HttpSolrServer(url);
// add
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery(query);
solrQuery.setStart(_start);
solrQuery.setRows(_nbDocuments);
solrQuery.setRequestHandler("query");
solrQuery.set("fl", "*,score"); // 設定fl參數，指定要傳回哪個field的資料，這裡設定所有field與score
if (_fq1 != "") {
solrQuery.addFilterQuery("ProductName:" + _fq1);
}
if (_fq2 != "") {
solrQuery.addFilterQuery("publishedDate:" + _fq2);
}
if (sort != null) {
solrQuery.addSortField(sort, ORDER.asc);
}
solrQuery.setRequestHandler("/browse");
response = server.query(solrQuery);
docs = response.getResults();
if (docs != null) {
System.out.println(
docs.getNumFound() + " documents found, " + docs.size() + " returned : ");
setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
JSONObject jsonObject = new JSONObject();
for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) {
Entry<String, Object> entry = it2.next();
if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串，方便前端呈現
Date_format = entry.getValue().toString();
jsonObject.put(entry.getKey(), Date_format);
} else {
// 一般情況
jsonObject.put(entry.getKey(), entry.getValue());
}
}
System.out.print("\n");
// 將總共找到幾筆資料存在jsonarray的最後面傳給前端
jsonObject.put("TotalResultFound", docs.getNumFound());
jsonArray.add(jsonObject);
}
}
} catch (SolrServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return jsonArray;
}
@Override
protected OntologyTerm createRootTerm(OntologySearcher ontologySearcher) {
try {
return new LazyLoadedOntologyTerm(ontologySearcher, "MI:0190", "Interaction Type");
} catch (SolrServerException e) {
e.printStackTrace();
addErrorMessage("Could not load the tree", "");
}
return null;
}
public void forceCommitPending() {
try {
server.commit();
} catch (SolrServerException ex) {
ex.printStackTrace();
Logger.getRootLogger().error(ex.getMessage());
} catch (IOException ex) {
ex.printStackTrace();
Logger.getRootLogger().error(ex.getMessage());
}
}
private void index(TruckEvent truckEvent) {
try {
LOG.info("Starting process to solr index document[" + truckEvent + "]");
UpdateResponse response = server.addBean(truckEvent);
LOG.info(
"Indexed document with id: " + truckEvent.getId() + " status: " + response.getStatus());
} catch (IOException e) {
LOG.error("Could not index document: " + truckEvent.getId() + " " + e.getMessage());
e.printStackTrace();
} catch (SolrServerException e) {
LOG.error("Could not index document: " + truckEvent.getId() + " " + e.getMessage());
e.printStackTrace();
}
}
public void run() {
while (true) {
try {
Thread.sleep(15000);
server.commit();
LOG.info("Committing Index");
} catch (InterruptedException e) {
LOG.error("Interrupted: " + e.getMessage());
e.printStackTrace();
} catch (SolrServerException e) {
LOG.error("Error committing: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
LOG.error("Error committing: " + e.getMessage());
e.printStackTrace();
}
}
}
/**
* Returns a MODS record for a given recordIdentifier.
*
* @param id a recordIdentifier for a solr document
* @return the ModsType for this recordidentifier
* @see ModsType
*/
public ModsType getMods(String id) throws JAXBException {
SolrDocumentList docs;
SolrDocument doc;
ModsType modsType = new ModsType();
HttpSolrServer server = null;
try {
server = SolrServer.getSolrConnection();
SolrQuery query = new SolrQuery("recordIdentifier:" + id);
QueryResponse response = server.query(query);
docs = response.getResults();
if (docs.size() == 0) throw new ResourceNotFoundException("Item " + id + " not found");
else {
doc = docs.get(0);
modsType = getModsType(doc);
}
} catch (SolrServerException se) {
se.printStackTrace();
log.error(se.getMessage());
}
return modsType;
}
// deprecated - use getModsType instead, rewrite and use this if we decide to wrap
// the mods document in an <item> tag
public Item getItem(String id) {
SolrDocumentList docs;
SolrDocument doc;
Item item = new Item();
HttpSolrServer server = null;
try {
server = SolrServer.getSolrConnection();
SolrQuery query = new SolrQuery("recordIdentifier:" + id);
QueryResponse response = server.query(query);
docs = response.getResults();
if (docs.size() == 0) item = null;
else {
doc = docs.get(0);
item = setItem(doc);
}
} catch (SolrServerException se) {
log.error(se.getMessage());
se.printStackTrace();
}
return item;
}
public boolean execute() {
SolrQuery query = queryGenerator.generate();
try {
logger.debug("executing query: " + query);
long init = System.nanoTime();
QueryResponse response = this.executeQuery(query);
long clientTime = (System.nanoTime() - init) / 1000000;
logger.debug(
response.getResults().getNumFound() + " results found in " + response.getQTime() + " ms");
if (response.getQTime() < 0) {
throw new RuntimeException(
"The query returned less than 0 as q time: "
+ response.getResponseHeader().get("q")
+ response.getQTime());
}
executor.notifyQueryExecuted(response, clientTime);
} catch (SolrServerException e) {
logger.error("Error on Query " + query);
e.printStackTrace();
executor.notifyError(new QueryException(e, query));
return false;
}
return true;
}
public void startElement(XmlPullParser parser, String wholeDoc) throws Exception {
String qName = parser.getName();
if (qName.equalsIgnoreCase("DOC")) {
if (currentDoc != null) {
throw new Exception("Nested Document taggs. Input file has problems.\n");
} else {
currentDoc = new SolrInputDocument();
switch (parser.getAttributeCount()) {
case 0:
category = "web";
break;
case 1:
category = "discussion";
baseID = parser.getAttributeValue(0);
currentDoc.addField("id", parser.getAttributeValue(0));
break;
case 2:
category = "news";
currentDoc.addField("id", parser.getAttributeValue(0));
// TODO: process news type attribute if needed
break;
}
currentDoc.addField("category", category);
currentDoc.addField("whole_text", wholeDoc);
if (category.equals("web")) {
// TODO: call clean
}
/*Object[] preprocess = Preprocessor.Tokenize(StripXMLTags.strip(wholeDoc).toString());
currentDoc.addField("offsets", preprocess[0]);
currentDoc.addField("tokens", preprocess[1]);
currentDoc.addField("tree", preprocess[2]);*/
}
} else if (qName.equalsIgnoreCase("DOCID")) {
// Should happen in web documents only
inDocId = true;
} else if (qName.equalsIgnoreCase("DOCTYPE")) {
// Should happen in web documents only
// TODO: process if this element is needed
// inDocType = true;
} else if (qName.equalsIgnoreCase("DATETIME")) {
inDate = true;
} else if (qName.equalsIgnoreCase("DATELINE")) {
inDate = true;
} else if (qName.equalsIgnoreCase("POSTDATE")) {
inDate = true;
} else if (qName.equalsIgnoreCase("P")) {
inContent = true;
} else if (qName.equalsIgnoreCase("headline")) {
inContent = true;
} else if (qName.equalsIgnoreCase("post")) {
assert (category.equals("discussion") || category.equals("web"));
assert (baseID != null);
if (currentDoc != null) { // Finish the previous doc
if (category.equals("web") && currentDoc.getFieldValues("id") == null) {
currentDoc.addField("id", baseID);
}
assert (currentDoc.getFieldValue("id") != null);
try {
SolrImporter.addDoc(server, currentDoc);
/*//Make sure offsets are correct
String rawText = SolrInterface.getRawDocument(baseID);
int diff = StripXMLTags.verify(wholeDoc, new StringBuffer(rawText));
if (diff!=-1){
System.out.println(diff+":'(");
}*/
} catch (SolrServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
currentDoc = null;
}
currentDoc = new SolrInputDocument();
currentDoc.addField("category", category);
if (parser.getAttributeCount() == 3
&& parser.getAttributeName(0).equalsIgnoreCase("author")) {
assert (category.equals("discussion"));
currentDoc.addField("author", parser.getAttributeValue(0));
// Append post id to doc id
currentDoc.addField("id", baseID + "." + parser.getAttributeValue(2));
} else {
assert (category.equals("web"));
// Append the poster number to the doc id
currentDoc.addField("id", baseID + "." + (count++));
}
// TODO: datetime for discussions if needed
inContent = true;
} else if (qName.equalsIgnoreCase("POSTER")) {
inPoster = true;
} else if (qName.equalsIgnoreCase("BODY")) {
// ignore since we populate fields inside
} else if (qName.equalsIgnoreCase("TEXT")) {
// ignore since we populate fields inside
} else if (qName.equalsIgnoreCase("docs")) {
// ignore
} else if (qName.equalsIgnoreCase("QUOTE")) {
// Note: Hack so that we have the things before the quote tag due to not XML compliant quote
// tag crashing parser
/*if (category.equalsIgnoreCase("web")){
//currentDoc.addField("content", contentBuffer.toString());
endElement("post");
}*/
} else if (qName.equals("a")) {
// ignore
} else if (qName.equals("img")) {
// ignore
} else {
System.out.println("Unexpected node:" + qName);
}
}
public NGDItem call() {
NGDItem totalResults = null;
SolrQuery query = new SolrQuery();
query.setQuery("+text:(*:*)");
query.addFilterQuery("+pub_date_year:[1993 TO 2013]");
query.setParam("fl", "pmid");
for (int i = 0; i < term1Array.size(); i++) {
String term1Combined = createCombinedTerm(term1Array.get(i));
query.addFilterQuery("+text:(" + term1Combined + ")");
}
for (int j = 0; j < term2Array.size(); j++) {
String term2Combined = createCombinedTerm(term2Array.get(j));
query.addFilterQuery("+text:(" + term2Combined + ")");
}
boolean retry = false;
try {
QueryResponse rsp = this.server.query(query);
totalResults =
new NGDItem(
this.term1count,
this.term2count,
this.term1,
this.term2,
this.term1Array,
this.term2Array,
rsp.getResults().getNumFound(),
useAlias,
this.totalDocCount);
} catch (SolrServerException e) {
log.warning(e.getMessage());
e.printStackTrace();
retry = true;
}
if (retry) {
log.warning("retrying query");
try {
// retry once.....
QueryResponse rsp = this.server.query(query);
totalResults =
new NGDItem(
this.term1count,
this.term2count,
this.term1,
this.term2,
this.term1Array,
this.term2Array,
rsp.getResults().getNumFound(),
useAlias,
this.totalDocCount);
} catch (SolrServerException e) {
log.warning(e.getMessage());
e.printStackTrace();
System.exit(1);
}
}
return totalResults;
}
