Example #1
0
  public JSONArray Search(String query, String sort) {
    SolrServer server;
    SolrDocumentList docs = null;
    String Date_format = null; // for Date格式轉換
    JSONArray jsonArray = new JSONArray();
    try {
      server = new HttpSolrServer(url);
      // add

      SolrQuery solrQuery = new SolrQuery();
      solrQuery.setQuery(query);
      solrQuery.setStart(_start);
      solrQuery.setRows(_nbDocuments);
      solrQuery.setRequestHandler("query");
      solrQuery.set("fl", "*,score"); // 設定fl參數,指定要傳回哪個field的資料,這裡設定所有field與score
      if (_fq1 != "") {
        solrQuery.addFilterQuery("ProductName:" + _fq1);
      }
      if (_fq2 != "") {
        solrQuery.addFilterQuery("publishedDate:" + _fq2);
      }
      if (sort != null) {
        solrQuery.addSortField(sort, ORDER.asc);
      }
      solrQuery.setRequestHandler("/browse");
      response = server.query(solrQuery);
      docs = response.getResults();
      if (docs != null) {
        System.out.println(
            docs.getNumFound() + " documents found, " + docs.size() + " returned : ");
        setResultNumber(docs.getNumFound(), docs.size()); // 設定目前回傳幾筆資料給前端
        for (int i = 0; i < docs.size(); i++) {
          SolrDocument doc = docs.get(i);
          JSONObject jsonObject = new JSONObject();
          for (Iterator<Entry<String, Object>> it2 = doc.iterator(); it2.hasNext(); ) {
            Entry<String, Object> entry = it2.next();
            if (entry.getKey().equals("publishedDate")) { // 將傳回的date格式轉為純字串,方便前端呈現
              Date_format = entry.getValue().toString();
              jsonObject.put(entry.getKey(), Date_format);
            } else {
              // 一般情況
              jsonObject.put(entry.getKey(), entry.getValue());
            }
          }
          System.out.print("\n");
          // 將總共找到幾筆資料存在jsonarray的最後面傳給前端
          jsonObject.put("TotalResultFound", docs.getNumFound());
          jsonArray.add(jsonObject);
        }
      }

    } catch (SolrServerException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return jsonArray;
  }
 @Test(dependsOnMethods = "testAddDocument")
 public void testBasicQuery() throws SolrServerException {
   SolrServer server = solrServerFactory.getServer();
   SolrParams params = new SolrQuery("*:*");
   SolrDocumentList results = server.query(params).getResults();
   assertTrue(results.getNumFound() > 0, "didn't return any results");
 }
Example #3
0
    protected List<Vector> processQueryResponse(QueryResponse resp) {
      NamedList<Object> response = resp.getResponse();

      NamedList<Object> termVectorsNL = (NamedList<Object>) response.get("termVectors");
      if (termVectorsNL == null)
        throw new RuntimeException(
            "No termVectors in response! "
                + "Please check your query to make sure it is requesting term vector information from Solr correctly.");

      List<Vector> termVectors = new ArrayList<Vector>(termVectorsNL.size());
      Iterator<Map.Entry<String, Object>> iter = termVectorsNL.iterator();
      while (iter.hasNext()) {
        Map.Entry<String, Object> next = iter.next();
        String nextKey = next.getKey();
        Object nextValue = next.getValue();
        if (nextValue instanceof NamedList) {
          NamedList nextList = (NamedList) nextValue;
          Object fieldTerms = nextList.get(field);
          if (fieldTerms != null && fieldTerms instanceof NamedList) {
            termVectors.add(
                SolrTermVector.newInstance(nextKey, hashingTF, (NamedList<Object>) fieldTerms));
          }
        }
      }

      SolrDocumentList docs = resp.getResults();
      totalDocs = docs.getNumFound();

      return termVectors;
    }
  public SearchResponse search(SearchRequest request) {
    SolrQuery query = new SolrQuery();
    query.setRows(request.getLimit());
    query.setStart(request.getOffset() * request.getLimit());
    BooleanQuery aggregate = new BooleanQuery();
    if (!StringUtils.isBlank(request.getText())) {
      TermQuery termQuery = new TermQuery(new Term("", request.getText()));
      aggregate.add(termQuery, BooleanClause.Occur.SHOULD);
    }
    if (!StringUtils.isBlank(request.getEventId())) {
      aggregate.add(
          new TermQuery(new Term("eventid", request.getEventId())), BooleanClause.Occur.MUST);
    }
    query.setQuery(aggregate.toString());

    log.info("QUERY IS: " + query.toString());
    try {
      QueryResponse queryResponse = solrServer.query(query);
      log.info("RESPONSE FROM QUERY WAS: " + queryResponse);
      SolrDocumentList results = queryResponse.getResults();
      log.info("RESULTS WAS: " + results);
      ArrayList<SearchResponse.Hit> hits = new ArrayList<SearchResponse.Hit>();
      for (SolrDocument result : results) {
        hits.add(
            new SearchResponse.Hit(
                ObjectType.session, String.valueOf(result.getFieldValue("id")), null));
      }
      return new SearchResponse(results.getNumFound(), queryResponse.getElapsedTime(), hits);
    } catch (SolrServerException e) {
      e.printStackTrace();
    }
    return null;
  }
Example #5
0
 /**
  * Returns SearchResultsSlim for a given SolrDocumentList. A "slimmer" results object without the
  * "items" wrapper element is created for better transform to json.
  *
  * @param doc solr document list to build results
  * @return the SearchResultsSlim object for this solr result
  * @see SearchResultsSlim
  */
 private SearchResultsSlim buildSlimResults(SolrDocumentList docs) {
   SearchResultsSlim results = new SearchResultsSlim();
   Pagination pagination = new Pagination();
   pagination.setNumFound(docs.getNumFound());
   pagination.setStart(docs.getStart());
   pagination.setRows(limit);
   // List<ModsType> modsTypes = new ArrayList<ModsType>();
   List<Item> items = new ArrayList<Item>();
   for (final SolrDocument doc : docs) {
     Item item = new Item();
     ModsType modsType = null;
     try {
       modsType = (new ItemDAO()).getModsType(doc);
     } catch (JAXBException je) {
       log.error(je.getMessage());
       je.printStackTrace();
     }
     item.setModsType(modsType);
     items.add(item);
   }
   results.setItems(items);
   results.setPagination(pagination);
   if (facet != null) results.setFacet(facet);
   return results;
 }
 @Test(dependsOnMethods = "testBasicQuery", dataProvider = "documentMap")
 public void testIdQuery(Map<String, String[]> document) throws SolrServerException {
   String id = document.get("id")[0];
   SolrServer server = solrServerFactory.getServer();
   SolrParams params = new SolrQuery("id:" + id);
   SolrDocumentList results = server.query(params).getResults();
   assertEquals(results.getNumFound(), 1, "didn't find article by id");
 }
Example #7
0
 public void writeSolrDocumentList(SolrDocumentList docs) throws IOException {
   writeTag(SOLRDOCLST);
   List<Number> l = new ArrayList<>(3);
   l.add(docs.getNumFound());
   l.add(docs.getStart());
   l.add(docs.getMaxScore());
   writeArray(l);
   writeArray(docs);
 }
  @Override
  public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
    productsToFilter = new ArrayList<String>();

    String queryString = "*:*";
    String sortCriteria = "user_count_purchased desc";
    ModifiableSolrParams solrParams = new ModifiableSolrParams();
    QueryResponse response = null;
    RecommendResponse searchResponse = new RecommendResponse();

    try {
      String filterQueryString =
          RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter);

      if (query.getProductIds() != null && query.getProductIds().size() > 0) {
        productsToFilter.addAll(query.getProductIds());
      }
      if (query.getUser() != null) {
        if (alreadyBoughtProducts != null) {
          productsToFilter.addAll(alreadyBoughtProducts);
        }
      }

      if (productsToFilter != null && productsToFilter.size() > 0) {
        if (filterQueryString.length() > 0) {
          filterQueryString += " OR ";
        }
        filterQueryString +=
            RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter);
      }
      solrParams.set("q", queryString);
      solrParams.set("fq", filterQueryString);
      solrParams.set("sort", sortCriteria);
      solrParams.set("rows", maxReuslts);

      response =
          SolrServiceContainer.getInstance()
              .getRecommendService()
              .getSolrServer()
              .query(solrParams);
      // fill response object
      List<String> extractedRecommendations =
          RecommendationQueryUtils.extractRecommendationIds(
              response.getBeans(CustomerAction.class));
      searchResponse.setResultItems(extractedRecommendations);
      searchResponse.setElapsedTime(response.getElapsedTime());
      SolrDocumentList docResults = response.getResults();
      searchResponse.setNumFound(docResults.getNumFound());
    } catch (SolrServerException e) {
      e.printStackTrace();
      searchResponse.setNumFound(0);
      searchResponse.setResultItems(new ArrayList<String>());
      searchResponse.setElapsedTime(-1);
    }

    return searchResponse;
  }
  @Override
  public void writeSolrDocumentList(
      String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
    boolean includeScore = false;
    if (fields != null) {
      includeScore = fields.contains("score");
      if (fields.size() == 0 || (fields.size() == 1 && includeScore) || fields.contains("*")) {
        fields = null; // null means return all stored fields
      }
    }

    int sz = docs.size();

    writeMapOpener(includeScore ? 4 : 3);
    incLevel();
    writeKey("numFound", false);
    writeLong(null, docs.getNumFound());
    writeMapSeparator();
    writeKey("start", false);
    writeLong(null, docs.getStart());

    if (includeScore && docs.getMaxScore() != null) {
      writeMapSeparator();
      writeKey("maxScore", false);
      writeFloat(null, docs.getMaxScore());
    }
    writeMapSeparator();
    // indent();
    writeKey("docs", false);
    writeArrayOpener(sz);

    incLevel();
    boolean first = true;

    SolrIndexSearcher searcher = req.getSearcher();
    for (SolrDocument doc : docs) {

      if (first) {
        first = false;
      } else {
        writeArraySeparator();
      }
      indent();
      writeSolrDocument(null, doc, fields, otherFields);
    }
    decLevel();
    writeArrayCloser();

    if (otherFields != null) {
      writeMap(null, otherFields, true, false);
    }

    decLevel();
    indent();
    writeMapCloser();
  }
  @Override
  public <T> ScoredPage<T> queryForPage(Query query, Class<T> clazz) {
    Assert.notNull(query, "Query must not be 'null'.");
    Assert.notNull(clazz, "Target class must not be 'null'.");

    QueryResponse response = query(query);
    List<T> beans = convertQueryResponseToBeans(response, clazz);
    SolrDocumentList results = response.getResults();
    return new SolrResultPage<T>(
        beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
  }
Example #11
0
  @Test
  public void testGetIdsWithParams() throws Exception {
    SolrDocumentList rsp =
        getSolrClient().getById(Arrays.asList("0", "1", "2"), params(CommonParams.FL, "id"));
    assertEquals(2, rsp.getNumFound());

    assertEquals("1", rsp.get(0).get("id"));
    assertNull("This field should have been removed from the response.", rsp.get(0).get("term_s"));
    assertNull("This field should have been removed from the response.", rsp.get(0).get("term2_s"));

    assertEquals("2", rsp.get(1).get("id"));
    assertNull("This field should have been removed from the response.", rsp.get(1).get("term_s"));
    assertNull("This field should have been removed from the response.", rsp.get(1).get("term2_s"));
  }
  /** SRN-96 */
  @Test
  public void testASCIIFoldingExpansion() throws IOException, SolrServerException {
    this.addJsonString("1", " { \"value\" : \"cafe\" } ");
    this.addJsonString("2", " { \"value\" : \"café\" } ");
    SolrQuery query = new SolrQuery();
    query.setQuery("cafe");
    query.setRequestHandler("keyword");
    query.setIncludeScore(true);

    // should match the two documents, with same score
    QueryResponse response = getWrapper().getServer().query(query);
    SolrDocumentList docList = response.getResults();
    assertEquals(2, docList.getNumFound());
    float score1 = (Float) docList.get(0).getFieldValue("score");
    float score2 = (Float) docList.get(1).getFieldValue("score");
    Assert.assertTrue("Score should be identical", score1 == score2);

    // should match the two documents, but should assign different score
    // id2 should receive better score than id1
    query = new SolrQuery();
    query.setQuery("café");
    query.setRequestHandler("keyword");
    query.setIncludeScore(true);

    response = getWrapper().getServer().query(query);
    docList = response.getResults();
    assertEquals(2, docList.getNumFound());
    if (docList.get(0).getFieldValue("url").equals("id1")) {
      score1 = (Float) docList.get(0).getFieldValue("score");
      score2 = (Float) docList.get(1).getFieldValue("score");
    } else {
      score2 = (Float) docList.get(0).getFieldValue("score");
      score1 = (Float) docList.get(1).getFieldValue("score");
    }
    Assert.assertTrue("id2 should get higher score than id1", score1 < score2);
  }
Example #13
0
  // 查询
  @Test
  public void testSolrForSearch() throws Exception {
    SolrQuery params = new SolrQuery();
    params.set("q", "*:*");
    QueryResponse response = solrServer.query(params);
    // 结果集
    SolrDocumentList docs = response.getResults();
    long count = docs.getNumFound();

    System.out.println("结果集有 :" + count);
    for (SolrDocument doc : docs) {
      String id = (String) doc.get("id");
      String name = (String) doc.get("name");
      System.out.println("id :" + id + "---" + "name :" + name);
    }
  }
  @Override
  public <T> HighlightPage<T> queryForHighlightPage(HighlightQuery query, Class<T> clazz) {
    Assert.notNull(query, "Query must not be 'null'.");
    Assert.notNull(clazz, "Target class must not be 'null'.");

    QueryResponse response = query(query);

    List<T> beans = convertQueryResponseToBeans(response, clazz);
    SolrDocumentList results = response.getResults();
    SolrResultPage<T> page =
        new SolrResultPage<T>(
            beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
    ResultHelper.convertAndAddHighlightQueryResponseToResultPage(response, page);

    return page;
  }
Example #15
0
  @Test
  public void testGetIds() throws Exception {
    SolrDocumentList rsp = getSolrClient().getById(Arrays.asList("0", "1", "2", "3", "4"));
    assertEquals(3, rsp.getNumFound());
    assertEquals("1", rsp.get(0).get("id"));
    assertEquals("Microsoft", rsp.get(0).get("term_s"));
    assertEquals("MSFT", rsp.get(0).get("term2_s"));

    assertEquals("2", rsp.get(1).get("id"));
    assertEquals("Apple", rsp.get(1).get("term_s"));
    assertEquals("AAPL", rsp.get(1).get("term2_s"));

    assertEquals("3", rsp.get(2).get("id"));
    assertEquals("Yahoo", rsp.get(2).get("term_s"));
    assertEquals("YHOO", rsp.get(2).get("term2_s"));
  }
  @Override
  public <T> FacetPage<T> queryForFacetPage(FacetQuery query, Class<T> clazz) {
    Assert.notNull(query, "Query must not be 'null'.");
    Assert.notNull(clazz, "Target class must not be 'null'.");

    QueryResponse response = query(query);

    List<T> beans = convertQueryResponseToBeans(response, clazz);
    SolrDocumentList results = response.getResults();
    SolrResultPage<T> page =
        new SolrResultPage<T>(
            beans, query.getPageRequest(), results.getNumFound(), results.getMaxScore());
    page.addAllFacetFieldResultPages(
        ResultHelper.convertFacetQueryResponseToFacetPageMap(query, response));
    page.addAllFacetPivotFieldResult(
        ResultHelper.convertFacetQueryResponseToFacetPivotMap(query, response));
    page.setFacetQueryResultPage(
        ResultHelper.convertFacetQueryResponseToFacetQueryResult(query, response));

    return page;
  }
  private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest)
      throws SolrServerException, IOException {
    // verify the join with fromIndex works
    final String fromQ = "match_s:c^2";
    CloudSolrClient client = cluster.getSolrClient();
    {
      final String joinQ =
          "{!join "
              + anyScoreMode(isScoresTest)
              + "from=join_s fromIndex="
              + fromColl
              + " to=join_s}"
              + fromQ;
      QueryRequest qr =
          new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
      QueryResponse rsp = new QueryResponse(client.request(qr), client);
      SolrDocumentList hits = rsp.getResults();
      assertTrue("Expected 1 doc, got " + hits, hits.getNumFound() == 1);
      SolrDocument doc = hits.get(0);
      assertEquals(toDocId, doc.getFirstValue("id"));
      assertEquals("b", doc.getFirstValue("get_s"));
      assertScore(isScoresTest, doc);
    }

    // negative test before creating an alias
    checkAbsentFromIndex(fromColl, toColl, isScoresTest);

    // create an alias for the fromIndex and then query through the alias
    String alias = fromColl + "Alias";
    CollectionAdminRequest.CreateAlias request =
        CollectionAdminRequest.createAlias(alias, fromColl);
    request.process(client);

    {
      final String joinQ =
          "{!join "
              + anyScoreMode(isScoresTest)
              + "from=join_s fromIndex="
              + alias
              + " to=join_s}"
              + fromQ;
      final QueryRequest qr =
          new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
      final QueryResponse rsp = new QueryResponse(client.request(qr), client);
      final SolrDocumentList hits = rsp.getResults();
      assertTrue("Expected 1 doc", hits.getNumFound() == 1);
      SolrDocument doc = hits.get(0);
      assertEquals(toDocId, doc.getFirstValue("id"));
      assertEquals("b", doc.getFirstValue("get_s"));
      assertScore(isScoresTest, doc);
    }

    // negative test after creating an alias
    checkAbsentFromIndex(fromColl, toColl, isScoresTest);

    {
      // verify join doesn't work if no match in the "from" index
      final String joinQ =
          "{!join "
              + (anyScoreMode(isScoresTest))
              + "from=join_s fromIndex="
              + fromColl
              + " to=join_s}match_s:d";
      final QueryRequest qr =
          new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
      final QueryResponse rsp = new QueryResponse(client.request(qr), client);
      final SolrDocumentList hits = rsp.getResults();
      assertTrue("Expected no hits", hits.getNumFound() == 0);
    }
  }
  /**
   * Sends an email to the given e-person with details of new items in the given dspace object (MUST
   * be a community or a collection), items that appeared yesterday. No e-mail is sent if there
   * aren't any new items in any of the dspace objects.
   *
   * @param context DSpace context object
   * @param eperson eperson to send to
   * @param rpkeys List of DSpace Objects
   * @param test
   * @throws SearchServiceException
   */
  public static void sendEmail(
      Researcher researcher,
      Context context,
      EPerson eperson,
      List<String> rpkeys,
      boolean test,
      List<String> relationFields)
      throws IOException, MessagingException, SQLException, SearchServiceException {

    CrisSearchService searchService = researcher.getCrisSearchService();

    // Get a resource bundle according to the eperson language preferences
    Locale supportedLocale = I18nUtil.getEPersonLocale(eperson);

    StringBuffer emailText = new StringBuffer();
    boolean isFirst = true;

    for (String rpkey : rpkeys) {
      SolrQuery query = new SolrQuery();
      query.setFields("search.resourceid");
      query.addFilterQuery(
          "{!field f=search.resourcetype}" + Constants.ITEM, "{!field f=inarchive}true");

      for (String tmpRelations : relationFields) {
        String fq = "{!field f=" + tmpRelations + "}" + rpkey;
        query.addFilterQuery(fq);
      }

      query.setRows(Integer.MAX_VALUE);

      if (ConfigurationManager.getBooleanProperty("eperson.subscription.onlynew", false)) {
        // get only the items archived yesterday
        query.setQuery("dateaccessioned:(NOW/DAY-1DAY)");
      } else {
        // get all item modified yesterday but not published the day
        // before
        // and all the item modified today and archived yesterday
        query.setQuery(
            "(item.lastmodified:(NOW/DAY-1DAY) AND dateaccessioned:(NOW/DAY-1DAY)) OR ((item.lastmodified:(NOW/DAY) AND dateaccessioned:(NOW/DAY-1DAY)))");
      }

      QueryResponse qResponse = searchService.search(query);
      SolrDocumentList results = qResponse.getResults();

      // Only add to buffer if there are new items
      if (results.getNumFound() > 0) {

        if (!isFirst) {
          emailText.append("\n---------------------------------------\n");
        } else {
          isFirst = false;
        }

        emailText
            .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.new-items", supportedLocale))
            .append(" ")
            .append(rpkey)
            .append(": ")
            .append(results.getNumFound())
            .append("\n\n");

        for (SolrDocument solrDoc : results) {

          Item item = Item.find(context, (Integer) solrDoc.getFieldValue("search.resourceid"));

          DCValue[] titles = item.getDC("title", null, Item.ANY);
          emailText
              .append("      ")
              .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.title", supportedLocale))
              .append(" ");

          if (titles.length > 0) {
            emailText.append(titles[0].value);
          } else {
            emailText.append(
                I18nUtil.getMessage("org.dspace.eperson.Subscribe.untitled", supportedLocale));
          }

          DCValue[] authors = item.getDC("contributor", Item.ANY, Item.ANY);

          if (authors.length > 0) {
            emailText
                .append("\n    ")
                .append(
                    I18nUtil.getMessage("org.dspace.eperson.Subscribe.authors", supportedLocale))
                .append(" ")
                .append(authors[0].value);

            for (int k = 1; k < authors.length; k++) {
              emailText.append("\n             ").append(authors[k].value);
            }
          }

          emailText
              .append("\n         ")
              .append(I18nUtil.getMessage("org.dspace.eperson.Subscribe.id", supportedLocale))
              .append(" ")
              .append(HandleManager.getCanonicalForm(item.getHandle()))
              .append("\n\n");
          context.removeCached(item, item.getID());
        }
      }
    }

    // Send an e-mail if there were any new items
    if (emailText.length() > 0) {

      if (test) {
        log.info(LogManager.getHeader(context, "subscription:", "eperson=" + eperson.getEmail()));
        log.info(LogManager.getHeader(context, "subscription:", "text=" + emailText.toString()));

      } else {

        Email email =
            ConfigurationManager.getEmail(
                I18nUtil.getEmailFilename(supportedLocale, "subscription"));
        email.addRecipient(eperson.getEmail());
        email.addArgument(emailText.toString());
        email.send();

        log.info(
            LogManager.getHeader(context, "sent_subscription", "eperson_id=" + eperson.getID()));
      }
    }
  }
Example #19
0
  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    response.setContentType(XML_RESPONSE_HEADER); // Talkback happens in XML form.
    response.setCharacterEncoding("UTF-8"); // Unicode++
    request.setCharacterEncoding("UTF-8");

    PrintWriter out = null; // The talkback buffer.

    // handle startrecord
    Integer startRecord = 0;

    if (!(request.getParameter("startRecord") == null)) {
      try {
        startRecord = Integer.parseInt(request.getParameter("startRecord")) - 1;
      } catch (NumberFormatException e) {
        startRecord = 0;
      }
    }

    // maximumrecords
    Integer maximumRecords = Integer.parseInt(this.config.getProperty("default_maximumRecords"));
    if (!(request.getParameter("maximumRecords") == null)) {
      maximumRecords = Integer.parseInt(request.getParameter("maximumRecords"));
    }

    // operation
    String operation = request.getParameter("operation");

    // x_collection
    String x_collection = request.getParameter("x-collection");
    if (x_collection == null) x_collection = this.config.getProperty("default_x_collection");
    if (x_collection == null) operation = null;

    // sortkeys
    String sortKeys = request.getParameter("sortKeys");

    // sortorder
    String sortOrder = request.getParameter("sortOrder");

    // recordschema
    String recordSchema = request.getParameter("recordSchema");
    if (recordSchema == null) recordSchema = "dc";

    if (recordSchema.equalsIgnoreCase("dcx")) {
      recordSchema = "dcx";
    }

    if (recordSchema.equalsIgnoreCase("solr")) {
      recordSchema = "solr";
    }

    // query request
    String query = request.getParameter("query");
    String q = request.getParameter("q");

    // who is requestor ?
    String remote_ip = request.getHeader("X-FORWARDED-FOR");

    if (remote_ip == null) {
      remote_ip = request.getRemoteAddr().trim();
    } else {
      remote_ip = request.getHeader("X-FORWARDED-FOR");
    }

    // handle debug
    Boolean debug = Boolean.parseBoolean(request.getParameter("debug"));
    if (!debug) {
      out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
    }

    // handle query
    if ((query == null) && (q != null)) {
      query = q;
    } else {
      if ((query != null) && (q == null)) {
        q = query;
      } else {
        operation = null;
      }
    }

    // handle operation
    if (operation == null) {
      if (query != null) {
        operation = "searchRetrieve";
      } else {
        operation = "explain";
      }
    }

    //  searchRetrieve
    if (operation.equalsIgnoreCase("searchRetrieve")) {
      if (query == null) {
        operation = "explain";
        log.debug(operation + ":" + query);
      }
    }

    // start talking back.
    String[] sq = {""};
    String solrquery = "";

    // facet

    String facet = null;
    List<FacetField> fct = null;

    if (request.getParameter("facet") != null) {
      facet = request.getParameter("facet");
      log.debug("facet : " + facet);
    }

    if (operation == null) {
      operation = "searchretrieve";
    } else { // explain response
      if (operation.equalsIgnoreCase("explain")) {
        log.debug("operation = explain");
        out.write("<srw:explainResponse xmlns:srw=\"http://www.loc.gov/zing/srw/\">");
        out.write("</srw:explainResponse>");
      } else { // DEBUG routine
        operation = "searchretrieve";

        String triplequery = null;

        if (query.matches(".*?\\[.+?\\].*?")) { // New symantic syntax
          triplequery = symantic_query(query);
          query = query.split("\\[")[0] + " " + triplequery;
          log.fatal(triplequery);

          solrquery = CQLtoLucene.translate(query, log, config);
        } else {
          solrquery = CQLtoLucene.translate(query, log, config);
        }
        log.debug(solrquery);

        if (debug == true) {
          response.setContentType(HTML_RESPONSE_HEADER);
          out = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), true);
          out.write("<html><body>\n\n");
          out.write("'" + remote_ip + "'<br>\n");
          out.write("<form action='http://www.kbresearch.nl/kbSRU'>");
          out.write("<input type=text name=q value='" + query + "' size=120>");
          out.write("<input type=hidden name=debug value=True>");
          out.write("<input type=submit>");
          out.write("<table border=1><tr><td>");
          out.write("q</td><td>" + query + "</td></tr><tr>");
          out.write("<td>query out</td><td>" + URLDecoder.decode(solrquery) + "</td></tr>");
          out.write(
              "<tr><td>SOLR_URL</td><td> <a href='"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/?q="
                  + solrquery
                  + "'>"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/select/?q="
                  + solrquery
                  + "</a><br>"
                  + this.config.getProperty("solr_url")
                  + solrquery
                  + "</td></tr>");
          out.write(
              "<b>SOLR_QUERY</b> : <BR> <iframe width=900 height=400 src='"
                  + this.config.getProperty(
                      "collection." + x_collection.toLowerCase() + ".solr_baseurl")
                  + "/../?q="
                  + solrquery
                  + "'></iframe><BR>");
          out.write(
              "<b>SRU_QUERY</b> : <BR> <a href="
                  + this.config.getProperty("baseurl")
                  + "?q="
                  + query
                  + "'>"
                  + this.config.getProperty("baseurl")
                  + "?q="
                  + query
                  + "</a><br><iframe width=901 height=400 src='http://www.kbresearch.nl/kbSRU/?q="
                  + query
                  + "'></iframe><BR>");
          out.write(
              "<br><b>JSRU_QUERY</b> : <BR><a href='http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection="
                  + x_collection
                  + "'>http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection=GGC</a><br><iframe width=900 height=400 src='http://jsru.kb.nl/sru/?query="
                  + query
                  + "&x-collection=GGC'></iframe>");

        } else { // XML SearchRetrieve response
          String url =
              this.config.getProperty("collection." + x_collection.toLowerCase() + ".solr_baseurl");
          String buffer = "";
          CommonsHttpSolrServer server = null;
          server = new CommonsHttpSolrServer(url);
          log.fatal("URSING " + url);
          server.setParser(new XMLResponseParser());
          int numfound = 0;
          try {
            SolrQuery do_query = new SolrQuery();
            do_query.setQuery(solrquery);
            do_query.setRows(maximumRecords);
            do_query.setStart(startRecord);

            if ((sortKeys != null) && (sortKeys.length() > 1)) {
              if (sortOrder != null) {
                if (sortOrder.equals("asc")) {
                  do_query.setSortField(sortKeys, SolrQuery.ORDER.asc);
                }
                if (sortOrder.equals("desc")) {
                  do_query.setSortField(sortKeys, SolrQuery.ORDER.desc);
                }
              } else {
                for (String str : sortKeys.trim().split(",")) {
                  str = str.trim();
                  if (str.length() > 1) {
                    if (str.equals("date")) {
                      do_query.setSortField("date_date", SolrQuery.ORDER.desc);
                      log.debug("SORTORDERDEBUG | DATE! " + str + " | ");
                      break;
                    } else {
                      do_query.setSortField(str + "_str", SolrQuery.ORDER.asc);
                      log.debug("SORTORDERDEBUG | " + str + " | ");
                      break;
                    }
                  }
                }
              }
            }

            if (facet != null) {
              if (facet.indexOf(",") > 1) {
                for (String str : facet.split(",")) {
                  if (str.indexOf("date") > 1) {
                    do_query.addFacetField(str);
                  } else {
                    do_query.addFacetField(str);
                  }
                  // do_query.setParam("facet.method", "enum");
                }
                // q.setFacetSort(false);
              } else {
                do_query.addFacetField(facet);
              }
              do_query.setFacet(true);
              do_query.setFacetMinCount(1);
              do_query.setFacetLimit(-1);
            }

            log.fatal(solrquery);

            QueryResponse rsp = null;
            boolean do_err = false;
            boolean do_sugg = false;
            SolrDocumentList sdl = null;
            String diag = "";
            StringBuffer suggest = new StringBuffer("");

            String content = "1";

            SolrQuery spellq = do_query;
            try {
              rsp = server.query(do_query);
            } catch (SolrServerException e) {
              String header = this.SRW_HEADER.replaceAll("\\$numberOfRecords", "0");
              out.write(header);
              diag = this.SRW_DIAG.replaceAll("\\$error", e.getMessage());
              do_err = true;
              rsp = null;
            }

            log.fatal("query done..");
            if (!(do_err)) { // XML dc response

              SolrDocumentList docs = rsp.getResults();
              numfound = (int) docs.getNumFound();
              int count = startRecord;
              String header =
                  this.SRW_HEADER.replaceAll("\\$numberOfRecords", Integer.toString(numfound));
              out.write(header);
              out.write("<srw:records>");

              Iterator<SolrDocument> iter = rsp.getResults().iterator();

              while (iter.hasNext()) {
                count += 1;
                if (recordSchema.equalsIgnoreCase("dc")) {
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                  out.write(
                      "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" xmlns:facets=\"info:srw/extension/4/facets\" >");
                  StringBuffer result = new StringBuffer("");

                  construct_lucene_dc(result, resultDoc);

                  out.write(result.toString());
                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }

                if (recordSchema.equalsIgnoreCase("solr")) {
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/solr</srw:recordSchema>");
                  out.write("<srw:recordData xmlns:expand=\"http://www.kbresearch.nl/expand\">");
                  StringBuffer result = new StringBuffer("");
                  construct_lucene_solr(result, resultDoc);
                  out.write(result.toString());

                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }

                if (recordSchema.equalsIgnoreCase("dcx")) { // XML dcx response
                  out.write("<srw:record>");
                  out.write("<srw:recordPacking>xml</srw:recordPacking>");
                  out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                  out.write(
                      "<srw:recordData><srw_dc:dc xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/marc.relators/\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                  SolrDocument resultDoc = iter.next();
                  content = (String) resultDoc.getFieldValue("id");

                  String dcx_data =
                      helpers.getOAIdcx(
                          "http://services.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
                          log);
                  if (x_collection.equalsIgnoreCase("ggc-thes")) {
                    dcx_data =
                        helpers.getOAIdcx(
                            "http://serviceso.kb.nl/mdo/oai?verb=GetRecord&identifier=" + content,
                            log);
                  }

                  if (!(dcx_data.length() == 0)) {
                    out.write(dcx_data);
                  } else {
                    // Should not do this!!

                    out.write("<srw:record>");
                    out.write("<srw:recordPacking>xml</srw:recordPacking>");
                    out.write("<srw:recordSchema>info:srw/schema/1/dc-v1.1</srw:recordSchema>");
                    out.write(
                        "<srw:recordData xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:mods=\"http://www.loc.gov/mods\" xmlns:dcterms=\"http://purl.org/dc/terms/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:dcx=\"http://krait.kb.nl/coop/tel/handbook/telterms.html\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:marcrel=\"http://www.loc.gov/loc.terms/relators/OTH\" >");
                    StringBuffer result = new StringBuffer("");

                    construct_lucene_dc(result, resultDoc);

                    out.write(result.toString());
                    out.write("</srw:recordData>");
                    out.write(
                        "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                    out.write("</srw:record>");
                  }

                  out.write("</srw_dc:dc>");

                  StringBuffer expand_data;
                  boolean expand = false;

                  if (content.startsWith("GGC-THES:AC:")) {
                    String tmp_content = "";
                    tmp_content = content.replaceFirst("GGC-THES:AC:", "");
                    log.fatal("calling get");
                    expand_data =
                        new StringBuffer(
                            helpers.getExpand(
                                "http://www.kbresearch.nl/general/lod_new/get/"
                                    + tmp_content
                                    + "?format=rdf",
                                log));
                    log.fatal("get finini");

                    if (expand_data.toString().length() > 4) {

                      out.write(
                          "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                      out.write(expand_data.toString());
                      expand = true;
                    }
                  } else {
                    expand_data =
                        new StringBuffer(
                            helpers.getExpand(
                                "http://www.kbresearch.nl/ANP.cgi?q=" + content, log));
                    if (expand_data.toString().length() > 0) {
                      if (!expand) {
                        out.write(
                            "<srw_dc:expand xmlns:srw_dc=\"info:srw/schema/1/dc-v1.1\" xmlns:expand=\"http://www.kbresearch.nl/expand\" xmlns:skos=\"http://www.w3.org/2004/02/skos/core#\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" >");
                        expand = true;
                      }
                      out.write(expand_data.toString());
                    }
                  }
                  if (expand) {
                    out.write("</srw_dc:expand>");
                  }

                  out.write("</srw:recordData>");
                  out.write(
                      "<srw:recordPosition>" + Integer.toString(count) + "</srw:recordPosition>");
                  out.write("</srw:record>");
                }
              }
            }

            if ((do_err) || (numfound == 0)) {
              log.fatal("I haz suggestions");

              try {
                spellq.setParam("spellcheck", true);
                spellq.setQueryType("/spell");
                server = new CommonsHttpSolrServer(url);
                rsp = server.query(spellq);
                sdl = rsp.getResults();
                SpellCheckResponse spell;
                spell = rsp.getSpellCheckResponse();
                List<SpellCheckResponse.Suggestion> suggestions = spell.getSuggestions();
                if (suggestions.isEmpty() == false) {
                  suggest.append("<srw:extraResponseData>");
                  suggest.append("<suggestions>");

                  for (SpellCheckResponse.Suggestion sugg : suggestions) {
                    suggest.append("<suggestionfor>" + sugg.getToken() + "</suggestionfor>");
                    for (String item : sugg.getSuggestions()) {
                      suggest.append("<suggestion>" + item + "</suggestion>");
                    }
                    suggest.append("</suggestions>");
                    suggest.append("</srw:extraResponseData>");
                  }
                  do_sugg = true;
                }
              } catch (Exception e) {
                rsp = null;
                // log.fatal(e.toString());
              }
              ;
            }
            ;

            if (!do_err) {
              if (facet != null) {

                try {
                  fct = rsp.getFacetFields();
                  out.write("<srw:facets>");

                  for (String str : facet.split(",")) {
                    out.write("<srw:facet>");
                    out.write("<srw:facetType>");
                    out.write(str);
                    out.write("</srw:facetType>");

                    for (FacetField f : fct) {
                      log.debug(f.getName());
                      // if (f.getName().equals(str+"_str") || (f.getName().equals(str+"_date")) ) {
                      List<FacetField.Count> facetEnties = f.getValues();
                      for (FacetField.Count fcount : facetEnties) {
                        out.write("<srw:facetValue>");
                        out.write("<srw:valueString>");
                        out.write(helpers.xmlEncode(fcount.getName()));
                        out.write("</srw:valueString>");
                        out.write("<srw:count>");
                        out.write(Double.toString(fcount.getCount()));
                        out.write("</srw:count>");
                        out.write("</srw:facetValue>");
                        //   }
                      }
                    }
                    out.write("</srw:facet>");
                  }
                  out.write("</srw:facets>");
                  startRecord += 1;
                } catch (Exception e) {
                }

                // log.fatal(e.toString()); }
              }
            } else {
              out.write(diag);
            }
            out.write("</srw:records>"); // SearchRetrieve response footer
            String footer = this.SRW_FOOTER.replaceAll("\\$query", helpers.xmlEncode(query));
            footer = footer.replaceAll("\\$startRecord", (startRecord).toString());
            footer = footer.replaceAll("\\$maximumRecords", maximumRecords.toString());
            footer = footer.replaceAll("\\$recordSchema", recordSchema);
            if (do_sugg) {
              out.write(suggest.toString());
            }
            out.write(footer);
          } catch (MalformedURLException e) {
            out.write(e.getMessage());
          } catch (IOException e) {
            out.write("TO ERR is Human");
          }
        }
      }
    }
    out.close();
  }
Example #20
0
  /** Main method of this example. */
  @Override
  public void runExample(ExampleDriver driver) throws Exception {
    long startMs = System.currentTimeMillis();

    CommandLine cli = driver.getCommandLine();

    // Size of index batch requests to Solr
    //		int batchSize = Integer.parseInt(cli.getOptionValue("batchSize", "500"));

    // Get a connection to Solr cloud using Zookeeper
    String zkHost = cli.getOptionValue("zkhost", ZK_HOST);
    String collectionName = cli.getOptionValue("collection", COLLECTION);
    int zkClientTimeout = Integer.parseInt(cli.getOptionValue("zkClientTimeout", "15000"));

    CloudSolrServer solr = new CloudSolrServer(zkHost);
    solr.setDefaultCollection(collectionName);
    solr.setZkClientTimeout(zkClientTimeout);
    solr.connect();

    int numSent = 0;
    int numSkipped = 0;
    int lineNum = 0;
    SolrInputDocument doc = null;
    String line = null;

    // read file line-by-line
    BufferedReader reader = new BufferedReader(driver.readFile("log"));
    driver.rememberCloseable(reader);

    LogFormat fmt = LogFormat.valueOf(cli.getOptionValue("format", "solr"));

    // process each sighting as a document
    while ((line = reader.readLine()) != null) {
      doc = parseNextDoc(line, ++lineNum, fmt);
      if (doc != null) {
        addDocWithRetry(solr, doc, 10);
        ++numSent;
      } else {
        ++numSkipped;
        continue;
      }

      if (lineNum % 1000 == 0) log.info(String.format("Processed %d lines.", lineNum));
    }

    // hard commit all docs sent
    solr.commit(true, true);

    float tookSecs = Math.round(((System.currentTimeMillis() - startMs) / 1000f) * 100f) / 100f;
    log.info(
        String.format(
            "Sent %d log messages (skipped %d) took %f seconds", numSent, numSkipped, tookSecs));

    // queries to demonstrate results of indexing
    SolrQuery solrQuery = new SolrQuery("*:*");
    solrQuery.setRows(0);
    QueryResponse resp = solr.query(solrQuery);
    SolrDocumentList hits = resp.getResults();
    log.info("Match all docs distributed query found " + hits.getNumFound() + " docs.");

    solrQuery.set("shards", "shard1");
    resp = solr.query(solrQuery);
    hits = resp.getResults();
    log.info(
        "Match all docs non-distributed query to shard1 found " + hits.getNumFound() + " docs.");

    solrQuery.set("shards", "shard2");
    resp = solr.query(solrQuery);
    hits = resp.getResults();
    log.info(
        "Match all docs non-distributed query to shard2 found " + hits.getNumFound() + " docs.");

    solr.shutdown();
  }
    public void merge(ResponseBuilder rb, ShardRequest sreq) {

      // id to shard mapping, to eliminate any accidental dups
      HashMap<Object, String> uniqueDoc = new HashMap<>();

      NamedList<Object> shardInfo = null;
      if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
        shardInfo = new SimpleOrderedMap<>();
        rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo);
      }

      IndexSchema schema = rb.req.getSchema();
      SchemaField uniqueKeyField = schema.getUniqueKeyField();

      long numFound = 0;
      Float maxScore = null;
      boolean partialResults = false;
      List<ShardDoc> shardDocs = new ArrayList();

      for (ShardResponse srsp : sreq.responses) {
        SolrDocumentList docs = null;

        if (shardInfo != null) {
          SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();

          if (srsp.getException() != null) {
            Throwable t = srsp.getException();
            if (t instanceof SolrServerException) {
              t = ((SolrServerException) t).getCause();
            }
            nl.add("error", t.toString());
            StringWriter trace = new StringWriter();
            t.printStackTrace(new PrintWriter(trace));
            nl.add("trace", trace.toString());
            if (srsp.getShardAddress() != null) {
              nl.add("shardAddress", srsp.getShardAddress());
            }
          } else {
            docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
            nl.add("numFound", docs.getNumFound());
            nl.add("maxScore", docs.getMaxScore());
            nl.add("shardAddress", srsp.getShardAddress());
          }
          if (srsp.getSolrResponse() != null) {
            nl.add("time", srsp.getSolrResponse().getElapsedTime());
          }

          shardInfo.add(srsp.getShard(), nl);
        }
        // now that we've added the shard info, let's only proceed if we have no error.
        if (srsp.getException() != null) {
          partialResults = true;
          continue;
        }

        if (docs == null) { // could have been initialized in the shards info block above
          docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
        }

        NamedList<?> responseHeader =
            (NamedList<?>) srsp.getSolrResponse().getResponse().get("responseHeader");
        if (responseHeader != null
            && Boolean.TRUE.equals(
                responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
          partialResults = true;
        }

        // calculate global maxScore and numDocsFound
        if (docs.getMaxScore() != null) {
          maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
        }
        numFound += docs.getNumFound();

        SortSpec ss = rb.getSortSpec();
        Sort sort = ss.getSort();

        NamedList sortFieldValues =
            (NamedList) (srsp.getSolrResponse().getResponse().get("merge_values"));
        NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
        List lst = (List) unmarshalledSortFieldValues.getVal(0);

        for (int i = 0; i < docs.size(); i++) {
          SolrDocument doc = docs.get(i);
          Object id = doc.getFieldValue(uniqueKeyField.getName());

          String prevShard = uniqueDoc.put(id, srsp.getShard());
          if (prevShard != null) {
            // duplicate detected
            numFound--;

            // For now, just always use the first encountered since we can't currently
            // remove the previous one added to the priority queue.  If we switched
            // to the Java5 PriorityQueue, this would be easier.
            continue;
            // make which duplicate is used deterministic based on shard
            // if (prevShard.compareTo(srsp.shard) >= 0) {
            //  TODO: remove previous from priority queue
            //  continue;
            // }
          }

          ShardDoc shardDoc = new ShardDoc();
          shardDoc.id = id;
          shardDoc.shard = srsp.getShard();
          shardDoc.orderInShard = i;
          Object scoreObj = lst.get(i);
          if (scoreObj != null) {
            shardDoc.score = ((Integer) scoreObj).floatValue();
          }
          shardDocs.add(shardDoc);
        } // end for-each-doc-in-response
      } // end for-each-response

      Collections.sort(
          shardDocs,
          new Comparator<ShardDoc>() {
            @Override
            public int compare(ShardDoc o1, ShardDoc o2) {
              if (o1.score < o2.score) {
                return 1;
              } else if (o1.score > o2.score) {
                return -1;
              } else {
                return 0; // To change body of implemented methods use File | Settings | File
                // Templates.
              }
            }
          });

      int resultSize = shardDocs.size();

      Map<Object, ShardDoc> resultIds = new HashMap<>();
      for (int i = 0; i < shardDocs.size(); i++) {
        ShardDoc shardDoc = shardDocs.get(i);
        shardDoc.positionInResponse = i;
        // Need the toString() for correlation with other lists that must
        // be strings (like keys in highlighting, explain, etc)
        resultIds.put(shardDoc.id.toString(), shardDoc);
      }

      // Add hits for distributed requests
      // https://issues.apache.org/jira/browse/SOLR-3518
      rb.rsp.addToLog("hits", numFound);

      SolrDocumentList responseDocs = new SolrDocumentList();
      if (maxScore != null) responseDocs.setMaxScore(maxScore);
      responseDocs.setNumFound(numFound);
      responseDocs.setStart(0);
      // size appropriately
      for (int i = 0; i < resultSize; i++) responseDocs.add(null);

      // save these results in a private area so we can access them
      // again when retrieving stored fields.
      // TODO: use ResponseBuilder (w/ comments) or the request context?
      rb.resultIds = resultIds;
      rb.setResponseDocs(responseDocs);

      if (partialResults) {
        rb.rsp
            .getResponseHeader()
            .add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
      }
    }
  @Override
  public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
    ModifiableSolrParams solrParams = new ModifiableSolrParams();
    QueryResponse response = null;
    RecommendResponse searchResponse = new RecommendResponse();

    long step0ElapsedTime = 0;
    long step1ElapsedTime;
    List<String> recommendations = new ArrayList<String>();

    try {
      // STEP 0 - get products from a user
      if (query.getUser() != null) {
        if (query.getProductIds() == null || query.getProductIds().size() == 0) {
          if (alreadyBoughtProducts != null) {
            query.setProductIds(alreadyBoughtProducts);
          } else {
          }
        }
      }

      solrParams = getInteractionsFromMeAndUSersThatILikedOrCommented(query.getUser());

      response =
          SolrServiceContainer.getInstance()
              .getSocialActionService()
              .getSolrServer()
              .query(solrParams);
      step1ElapsedTime = response.getElapsedTime();

      List<SocialAction> socialUsers = response.getBeans(SocialAction.class);

      if (socialUsers.size() == 0) {
        searchResponse.setNumFound(0);
        searchResponse.setResultItems(recommendations);
        searchResponse.setElapsedTime(-1);
        return searchResponse;
      }

      SocialAction currentUserInteractions = socialUsers.get(0);
      if (currentUserInteractions.getUserId().equals(query.getUser())) {
        socialUsers.remove(0);
      } else {
        currentUserInteractions = null;
      }

      final Map<String, Integer> userInteractionMap = new HashMap<String, Integer>();

      if (currentUserInteractions != null) {
        List<String> usersThatPostedASnapshopToMe =
            currentUserInteractions.getUsersThatPostedASnapshopToMe();

        if (usersThatPostedASnapshopToMe != null) {
          fillInteractions(usersThatPostedASnapshopToMe, userInteractionMap);
        }
      }

      for (SocialAction socialUser : socialUsers) {
        Integer userInteraction = userInteractionMap.get(socialUser.getUserId());
        if (userInteraction == null) {
          userInteraction = 0;
        }

        if (socialUser.getUsersThatPostedASnapshopToMe() != null) {
          userInteraction +=
              Collections.frequency(socialUser.getUsersThatPostedASnapshopToMe(), query.getUser());
        }

        userInteractionMap.put(socialUser.getUserId(), userInteraction);
      }

      Comparator<String> interactionCountComparator =
          new Comparator<String>() {

            @Override
            public int compare(String a, String b) {
              if (userInteractionMap.get(a) > userInteractionMap.get(b)) {
                return -1;
              } else if (userInteractionMap.get(a).equals(userInteractionMap.get(b))) {
                return 0;
              } else {
                return 1;
              }
            }
          };

      TreeMap<String, Integer> sorted_map =
          new TreeMap<String, Integer>(interactionCountComparator);
      sorted_map.putAll(userInteractionMap);
      solrParams = getSTEP2Params(query, maxReuslts, sorted_map);
      // TODO Facet for confidence value
      response =
          SolrServiceContainer.getInstance().getResourceService().getSolrServer().query(solrParams);
      // fill response object
      List<Resource> beans = response.getBeans(Resource.class);
      searchResponse.setResultItems(RecommendationQueryUtils.extractRecommendationIds(beans));
      searchResponse.setElapsedTime(
          step0ElapsedTime + step1ElapsedTime + response.getElapsedTime());

      SolrDocumentList docResults = response.getResults();
      searchResponse.setNumFound(docResults.getNumFound());
    } catch (Exception e) {
      System.out.println(solrParams);
      e.printStackTrace();
      searchResponse.setNumFound(0);
      searchResponse.setResultItems(recommendations);
      searchResponse.setElapsedTime(-1);
    }

    return searchResponse;
  }
Example #23
0
  public static serverObjects respond(
      @SuppressWarnings("unused") final RequestHeader header,
      final serverObjects post,
      final serverSwitch env) {
    // return variable that accumulates replacements
    final Switchboard sb = (Switchboard) env;
    final serverObjects prop = new serverObjects();
    final Segment segment = sb.index;
    final SolrConnector connector = segment.fulltext().getDefaultConnector();

    // avoid UNRESOLVED PATTERN
    prop.put("url", "");
    prop.put("citations", 0);
    prop.put("sentences", 0);

    DigestURL uri = null;
    String url = "";
    String hash = "";
    int ch = 10;
    boolean filter = false; // show cited sentences only
    if (post != null) {
      if (post.containsKey("url")) {
        url = post.get("url");
        if (!url.startsWith("http://")
            && !url.startsWith("https://")
            && !url.startsWith("ftp://")
            && !url.startsWith("smb://")
            && !url.startsWith("file://")) {
          url = "http://" + url;
        }
      }
      if (post.containsKey("hash")) {
        hash = post.get("hash");
      }
      if (post.containsKey("ch")) {
        ch = post.getInt("ch", ch);
      }
      filter = post.getBoolean("filter");
    }
    prop.put("filter", filter);
    if (url.length() > 0) {
      try {
        uri = new DigestURL(url, null);
        hash = ASCII.String(uri.hash());
      } catch (final MalformedURLException e) {
      }
    }
    if (uri == null && hash.length() > 0) {
      try {
        uri = sb.getURL(ASCII.getBytes(hash));
        if (uri == null) {
          connector.commit(true); // try again, that url can be fresh
          uri = sb.getURL(ASCII.getBytes(hash));
        }
      } catch (IOException e) {
        ConcurrentLog.logException(e);
      }
    }
    if (uri == null) return prop; // no proper url addressed
    url = uri.toNormalform(true);
    prop.put("url", url);

    // get the document from the index
    SolrDocument doc;
    try {
      doc =
          segment
              .fulltext()
              .getDefaultConnector()
              .getDocumentById(
                  hash,
                  CollectionSchema.title.getSolrFieldName(),
                  CollectionSchema.text_t.getSolrFieldName());
    } catch (final IOException e1) {
      return prop;
    }
    @SuppressWarnings("unchecked")
    ArrayList<String> title =
        (ArrayList<String>) doc.getFieldValue(CollectionSchema.title.getSolrFieldName());
    String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());

    ArrayList<String> sentences = new ArrayList<String>();
    if (title != null) for (String s : title) if (s.length() > 0) sentences.add(s);
    if (text != null && !text.isEmpty()) {
      SentenceReader sr = new SentenceReader(text);
      StringBuilder line;
      while (sr.hasNext()) {
        line = sr.next();
        if (line.length() > 0) sentences.add(line.toString());
      }
    }

    // for each line make a statistic about the number of occurrences somewhere else
    OrderedScoreMap<String> scores =
        new OrderedScoreMap<String>(null); // accumulates scores for citating urls
    LinkedHashMap<String, Set<DigestURL>> sentenceOcc = new LinkedHashMap<String, Set<DigestURL>>();
    for (String sentence : sentences) {
      if (sentence == null || sentence.length() < 40) {
        // do not count the very short sentences
        sentenceOcc.put(sentence, null);
        continue;
      }
      try {
        sentence = sentence.replace('"', '\'');
        SolrDocumentList doclist =
            connector.getDocumentListByQuery(
                "text_t:\"" + sentence + "\"",
                CollectionSchema.url_chars_i.getSolrFieldName() + " asc",
                0,
                100,
                CollectionSchema.sku.getSolrFieldName());
        int count = (int) doclist.getNumFound();
        if (count > 0) {
          Set<DigestURL> list = new TreeSet<DigestURL>();
          for (SolrDocument d : doclist) {
            String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName());
            if (u == null || u.equals(url)) continue;
            scores.inc(u);
            try {
              list.add(new DigestURL(u, null));
            } catch (final MalformedURLException e) {
            }
          }
          sentenceOcc.put(sentence, list);
        }
      } catch (final Throwable ee) {

      }
    }
    sentences.clear(); // we do not need this again

    // iterate the sentences
    int i = 0;
    int sentenceNr = 0;
    for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) {
      Set<DigestURL> app = se.getValue();
      if (filter) { // prepare list, only include sentence with citation
        if (app != null && app.size() > 0) {
          StringBuilder dd = new StringBuilder(se.getKey());
          prop.put("sentences_" + i + "_dt", sentenceNr);
          dd.append("<br/>appears in:");
          for (DigestURL u : app) {
            if (u != null) {
              dd.append(" <a href=\"")
                  .append(u.toNormalform(false))
                  .append("\">")
                  .append(u.getHost())
                  .append("</a>");
            }
          }
          prop.put("sentences_" + i + "_dd", dd.toString());
          i++;
        }
      } else { // prepare list, include all sentences
        StringBuilder dd = new StringBuilder(se.getKey());
        prop.put("sentences_" + i + "_dt", sentenceNr);
        if (app != null && app.size() > 0) {
          dd.append("<br/>appears in:");
          for (DigestURL u : app) {
            if (u != null) {
              dd.append(" <a href=\"")
                  .append(u.toNormalform(false))
                  .append("\">")
                  .append(u.getHost())
                  .append("</a>");
            }
          }
        }
        prop.put("sentences_" + i + "_dd", dd.toString());
        i++;
      }
      sentenceNr++;
    }
    prop.put("sentences", i);

    // iterate the citations in order of number of citations
    i = 0;
    for (String u : scores.keyList(false)) {
      try {
        DigestURL uu = new DigestURL(u, null);
        prop.put("citations_" + i + "_dt", "<a href=\"" + u + "\">" + u + "</a>");
        StringBuilder dd = new StringBuilder();
        dd.append("makes ")
            .append(Integer.toString(scores.get(u)))
            .append(" citations: of ")
            .append(url);
        for (Map.Entry<String, Set<DigestURL>> se : sentenceOcc.entrySet()) {
          Set<DigestURL> occurls = se.getValue();
          if (occurls != null && occurls.contains(uu))
            dd.append("<br/><a href=\"/solr/select?q=text_t:%22")
                .append(se.getKey().replace('"', '\''))
                .append("%22&rows=100&grep=&wt=grephtml\">")
                .append(se.getKey())
                .append("</a>");
        }
        prop.put("citations_" + i + "_dd", dd.toString());
        i++;
      } catch (final MalformedURLException e) {
      }
    }
    prop.put("citations", i);

    // find similar documents from different hosts
    i = 0;
    for (String u : scores.keyList(false)) {
      if (scores.get(u) < ch) continue;
      try {
        DigestURL uu = new DigestURL(u, null);
        if (uu.getOrganization().equals(uri.getOrganization())) continue;
        prop.put("similar_links_" + i + "_url", u);
        i++;
      } catch (final MalformedURLException e) {
      }
    }
    prop.put("similar_links", i);
    prop.put("similar", i > 0 ? 1 : 0);

    // return rewrite properties
    return prop;
  }
Example #24
0
  private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
    SortSpec ss = rb.getSortSpec();
    Sort sort = ss.getSort();

    SortField[] sortFields = null;
    if (sort != null) sortFields = sort.getSort();
    else {
      sortFields = new SortField[] {SortField.FIELD_SCORE};
    }

    SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();

    // id to shard mapping, to eliminate any accidental dups
    HashMap<Object, String> uniqueDoc = new HashMap<Object, String>();

    // Merge the docs via a priority queue so we don't have to sort *all* of the
    // documents... we only need to order the top (rows+start)
    ShardFieldSortedHitQueue queue;
    queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());

    long numFound = 0;
    Float maxScore = null;
    for (ShardResponse srsp : sreq.responses) {
      SolrDocumentList docs =
          (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");

      // calculate global maxScore and numDocsFound
      if (docs.getMaxScore() != null) {
        maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
      }
      numFound += docs.getNumFound();

      NamedList sortFieldValues =
          (NamedList) (srsp.getSolrResponse().getResponse().get("sort_values"));

      // go through every doc in this response, construct a ShardDoc, and
      // put it in the priority queue so it can be ordered.
      for (int i = 0; i < docs.size(); i++) {
        SolrDocument doc = docs.get(i);
        Object id = doc.getFieldValue(uniqueKeyField.getName());

        String prevShard = uniqueDoc.put(id, srsp.getShard());
        if (prevShard != null) {
          // duplicate detected
          numFound--;

          // For now, just always use the first encountered since we can't currently
          // remove the previous one added to the priority queue.  If we switched
          // to the Java5 PriorityQueue, this would be easier.
          continue;
          // make which duplicate is used deterministic based on shard
          // if (prevShard.compareTo(srsp.shard) >= 0) {
          //  TODO: remove previous from priority queue
          //  continue;
          // }
        }

        ShardDoc shardDoc = new ShardDoc();
        shardDoc.id = id;
        shardDoc.shard = srsp.getShard();
        shardDoc.orderInShard = i;
        Object scoreObj = doc.getFieldValue("score");
        if (scoreObj != null) {
          if (scoreObj instanceof String) {
            shardDoc.score = Float.parseFloat((String) scoreObj);
          } else {
            shardDoc.score = (Float) scoreObj;
          }
        }

        shardDoc.sortFieldValues = sortFieldValues;

        queue.insertWithOverflow(shardDoc);
      } // end for-each-doc-in-response
    } // end for-each-response

    // The queue now has 0 -> queuesize docs, where queuesize <= start + rows
    // So we want to pop the last documents off the queue to get
    // the docs offset -> queuesize
    int resultSize = queue.size() - ss.getOffset();
    resultSize = Math.max(0, resultSize); // there may not be any docs in range

    Map<Object, ShardDoc> resultIds = new HashMap<Object, ShardDoc>();
    for (int i = resultSize - 1; i >= 0; i--) {
      ShardDoc shardDoc = (ShardDoc) queue.pop();
      shardDoc.positionInResponse = i;
      // Need the toString() for correlation with other lists that must
      // be strings (like keys in highlighting, explain, etc)
      resultIds.put(shardDoc.id.toString(), shardDoc);
    }

    SolrDocumentList responseDocs = new SolrDocumentList();
    if (maxScore != null) responseDocs.setMaxScore(maxScore);
    responseDocs.setNumFound(numFound);
    responseDocs.setStart(ss.getOffset());
    // size appropriately
    for (int i = 0; i < resultSize; i++) responseDocs.add(null);

    // save these results in a private area so we can access them
    // again when retrieving stored fields.
    // TODO: use ResponseBuilder (w/ comments) or the request context?
    rb.resultIds = resultIds;
    rb._responseDocs = responseDocs;
  }
Example #25
0
  @RequestMapping(value = "/searchtweetresponse", method = RequestMethod.GET)
  @ResponseBody
  public String getTweets(
      @RequestParam("search") String searchKey,
      @RequestParam("start") int start,
      @RequestParam("end") int end) {

    logger.info("Welcome to Tweets");
    JSONObject finalObject = new JSONObject();

    int startRow = 0;
    int endRow = 10;
    // String tweetType="*";

    if (start > startRow) {
      startRow = start;
    }
    if (end > endRow) {
      endRow = end;
    }
    SolrQuery query = new SolrQuery("tweet_content:*" + searchKey + "*");
    query.setStart(startRow);
    query.setRows(endRow);
    HomeBean bean = new HomeBean();
    Collection<JSONObject> objectList = new ArrayList<JSONObject>();
    try {
      QueryResponse response = homeservice.getServiceResponse(query);
      if (response != null) {
        SolrDocumentList responseList = response.getResults();
        long totalCount = responseList.getNumFound();
        if (totalCount > 0) {
          bean.setResultCount(totalCount);
        }
        finalObject.put("total", totalCount);
        for (SolrDocument document : responseList) {
          JSONObject json = new JSONObject();
          json.put("id", document.getFieldValue("id"));
          json.put("tweet", document.getFieldValue("tweet_content"));
          objectList.add(json);
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
    }

    finalObject.put("tweetList", objectList);

    query.set("facet", true);
    query.addFacetField("tweetPostedTime");
    query.addSort("tweetPostedTime", SolrQuery.ORDER.asc);
    query.setRows(0);
    QueryResponse response = homeservice.getServiceResponse(query);
    // 2015-07-30T12:10:31Z
    DateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd");
    SimpleDateFormat newFormat = new SimpleDateFormat("MM-dd-yyyy");
    DateFormat dateFormat2 = new SimpleDateFormat("E MMM d hh:mm:ss zzz yyyy");
    List<Date> myList = new ArrayList<Date>();
    // Collection<JSONObject> tempList = new ArrayList<JSONObject>();
    Collection<JSONObject> dateCountList = new ArrayList<JSONObject>();
    JSONObject tempObj = new JSONObject();
    try {
      if (response != null) {
        FacetField timeFacet = response.getFacetField("tweetPostedTime");
        List<Count> list = timeFacet.getValues();
        Iterator itr = list.iterator();
        // Collection<String> productKeys = new ArrayList<String>();
        // Collection<String> smartPhoneKeys = new ArrayList<String>();
        while (itr.hasNext()) {
          Count timeObj = (Count) itr.next();
          if (timeObj != null) {
            String timeName = timeObj.getName();
            String originalTime = timeName.split("T")[0];
            Date beforeParse = dateFormat1.parse(originalTime);
            myList.add(beforeParse);
            JSONObject timeObject = new JSONObject();
            timeObject.put("date", newFormat.format(beforeParse));
            timeObject.put("count", timeObj.getCount());
            // tempList.add(timeObject);
            tempObj.put(newFormat.format(beforeParse), timeObject);
          }
        }
      }
      Collections.sort(myList);
      Iterator<Date> itr = myList.iterator();
      while (itr.hasNext()) {

        Date inputDate = dateFormat2.parse(itr.next().toString());

        String tempKey = newFormat.format(inputDate);
        System.out.println(tempKey);
        JSONObject newObj = (JSONObject) tempObj.get(tempKey);
        dateCountList.add(newObj);
      }
      finalObject.put("dateList", dateCountList);

    } catch (Exception e) {
      logger.error(e.getMessage());
      e.printStackTrace();
    }
    logger.info("bye Tweets");
    return finalObject.toString();
  }