Esempio n. 1
0
  public static void main(String a[]) throws MalformedURLException, SolrServerException {
    SolrServer hotServer = new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds");
    SolrQuery solrLocrQuery = new SolrQuery();
    /*solrLocrQuery.setParam("pos", "ORB");
    solrLocrQuery.setParam("locale", "en_US");*/
    solrLocrQuery.setRows(300);
    QueryResponse queryLocResponse = hotServer.query(solrLocrQuery);
    SolrDocumentList locList = queryLocResponse.getResults();

    for (SolrDocument doc : locList) {
      try {
        String str = doc.get("text_s").toString();
        if (str.toLowerCase().contains("price")) {
          if (str.toLowerCase().contains("assurance")) {
            String id = doc.get("id").toString();
            String url = doc.get("title_s").toString();

            System.out.println(id + "|" + url);
          }
        }
      } catch (Exception e) {

      }
    }
  }
 @Override
 public Pair<DocumentReference, String> next() {
   SolrDocument result = getResults().get(index++);
   String wiki = (String) result.get(FieldUtils.WIKI);
   String space = (String) result.get(FieldUtils.SPACE);
   String name = (String) result.get(FieldUtils.NAME);
   String locale = (String) result.get(FieldUtils.DOCUMENT_LOCALE);
   String version = (String) result.get(FieldUtils.VERSION);
   DocumentReference documentReference = new DocumentReference(wiki, space, name);
   if (!locale.isEmpty()) {
     documentReference = new DocumentReference(documentReference, LocaleUtils.toLocale(locale));
   }
   return new ImmutablePair<DocumentReference, String>(documentReference, version);
 }
Esempio n. 3
0
  // 查询
  @Test
  public void testSolrForSearch() throws Exception {
    SolrQuery params = new SolrQuery();
    params.set("q", "*:*");
    QueryResponse response = solrServer.query(params);
    // 结果集
    SolrDocumentList docs = response.getResults();
    long count = docs.getNumFound();

    System.out.println("结果集有 :" + count);
    for (SolrDocument doc : docs) {
      String id = (String) doc.get("id");
      String name = (String) doc.get("name");
      System.out.println("id :" + id + "---" + "name :" + name);
    }
  }
 @Override
 public Pair<DocumentReference, String> next() {
   SolrDocument result = getResults().get(index++);
   DocumentReference documentReference = this.solrDocumentReferenceResolver.resolve(result);
   String version = (String) result.get(FieldUtils.VERSION);
   return new ImmutablePair<DocumentReference, String>(documentReference, version);
 }
Esempio n. 5
0
  @Override
  public List<Message> GetResult(String word) {
    List<Message> messages = new ArrayList<Message>();
    try {
      sQuery.setQuery(word);
      QueryResponse response = solr.query(sQuery);
      SolrDocumentList reList = response.getResults();

      for (SolrDocument s : reList) {
        String id = (String) s.get("id");
        String des = (String) s.get("description");
        String s1 = s.get("title").toString();
        messages.add(new Message(id, des, s1));
      }
      return messages;
    } catch (SolrServerException | IOException e) {
      e.printStackTrace();
      return null;
    }
  }
  /**
   * Extract an {@link EntityReference} from the given {@link SolrDocument} (e.g. search result).
   * The entity type is inferred from the "type" field which must be specified and must have a valid
   * value (that corresponds to an existing {@link EntityType}).
   *
   * @param document a {@link SolrDocument} to extract the {@link EntityReference} from (the "type"
   *     field must be specified)
   * @param parameters the parameters to pass to the reference resolver (e.g. in case some reference
   *     components are missing)
   * @return the reference to the entity associated with the given {@link SolrDocument}
   */
  public EntityReference resolve(SolrDocument document, Object... parameters) {
    EntityType type;
    try {
      type = EntityType.valueOf((String) document.get(FieldUtils.TYPE));
    } catch (IllegalArgumentException e) {
      return null;
    } catch (NullPointerException e) {
      return null;
    }

    return resolve(document, type, parameters);
  }
Esempio n. 7
0
 public static boolean isLocalDirectory(SolrDocument doc) {
   if (doc.containsKey(TITLE)) {
     Object title = doc.get(TITLE);
     String titleContents = "";
     if (title instanceof ArrayList) {
       titleContents = (String) ((ArrayList) title).get(0);
     } else if (title instanceof String) {
       titleContents = (String) title;
     }
     return titleContents.startsWith("Index of /");
   }
   return false;
 }
Esempio n. 8
0
  @Override
  public boolean hasProperty(String name, String value) {
    List<String> fieldValues = asStringList(doc.get(name));
    if (fieldValues != null) {
      for (String fieldValue : fieldValues) {
        if (value.equals(fieldValue)) {
          return true;
        }
      }
    }

    return false;
  }
 public void run() {
   List<DBObject> conversations = conversationsRepository.getConversations();
   for (DBObject conversation : conversations) {
     List<SolrDocument> associatedTweets =
         rootTweetSolrRepository.findAssociatedTweets(
             valueOf(conversation.get("text")),
             valueOf(conversation.get("url")),
             valueOf(conversation.get("id")));
     List<Map> associatedTweetsInfo = new ArrayList<>();
     for (SolrDocument associatedTweet : associatedTweets) {
       associatedTweetsInfo.add(
           of(
               "text",
               associatedTweet.get("text"),
               "id",
               associatedTweet.get("id"),
               "url",
               associatedTweet.get("url") == null ? "" : associatedTweet.get("url")));
     }
     conversation.put("associated_tweets", associatedTweetsInfo);
   }
   conversationsRepository.update(conversations);
 }
Esempio n. 10
0
  public SolrRecord(SolrDocument doc, List<String> fields) {

    JSONObject jsonObject = new JSONObject();
    for (String s : fields) {
      if (doc.containsKey(s)) {
        jsonObject.put(s, doc.get(s));
      }
    }
    String contentStr = jsonObject.toString(3).replaceAll("\n", DEFAULT_NEWLINE);
    content = contentStr.getBytes();
    contentType = SolrUtils.getDocumentContentType(doc);
    id = getFieldStringValue(doc, ID, "not_found");
    fileName = new File(id).getName();
  }
Esempio n. 11
0
  private void writeSolr(SpectralClustering cluster) {
    Map<Integer, List<Doc>> map = new HashMap<Integer, List<Doc>>();
    int[] lab = cluster.getClusterLabel();
    for (int x = 0; x < lab.length; x++) {
      final Doc doc = list.get(x);
      if (!map.containsKey(new Integer(lab[x]))) {
        map.put(
            new Integer(lab[x]),
            new ArrayList<Doc>() {
              {
                add(doc);
              }
            });
      } else {
        map.get(new Integer(lab[x])).add(doc);
      }
    }
    for (Map.Entry<Integer, List<Doc>> e : map.entrySet()) {
      System.out.println("type:" + e.getKey());

      if (e.getValue().size() < 10) {
        StringBuffer query = new StringBuffer();
        for (Doc doc : e.getValue()) {
          query.append("url:\"").append(doc.url).append("\"");
          query.append(" OR ");
        }
        query.delete(query.lastIndexOf("OR"), query.length());
        String groupId = UUID.nameUUIDFromBytes(query.toString().getBytes()).toString();
        List<Map<String, Object>> toSave = new ArrayList<Map<String, Object>>();
        System.out.println("group:" + groupId + " = " + e.getValue().size());
        List<SolrDocument> list =
            (List<SolrDocument>) indexDao.sortList(query.toString(), 1, 100, "infoTime_dt desc");
        Date newest = new Date(0);
        int useful = 1;
        for (SolrDocument doc : list) {
          Date date = (Date) doc.get("infoTime_dt");
          if (date.getTime() > newest.getTime()) {}

          Map<String, Object> inputDoc = new HashMap<String, Object>(doc);
          inputDoc.put("useful_i", useful);
          inputDoc.put("sim_i", list.size());
          inputDoc.put("group_s", groupId);
          toSave.add(inputDoc);
          useful = 0;
        }
        indexDao.addIndex(toSave);
      }
    }
  }
 @SuppressWarnings("unused")
 private void printHits(SolrQuery q) throws SolrServerException {
   System.out.println("--- solr contents " + q.toString() + " ---");
   SolrDocumentList results = getSolr().query(q).getResults();
   if (results.size() == 0) {
     System.out.println("empty");
   }
   for (SolrDocument d : results) {
     for (String f : d.getFieldNames()) {
       String v = "" + d.get(f);
       System.out.print(", " + f + ": " + v);
     }
     System.out.println("");
   }
 }
  public boolean isSame(SolrIndexSearchMailDocument o) {
    if (this == o) {
      return true; // objects are the same
    }
    if (null == o) {
      return false; // dest is null
    }
    if (m_doc != null && o.m_doc != null) {
      if (m_doc.size() != o.m_doc.size()) {
        return false; // docs have different number of elements
      }
      for (String fieldName : m_doc.getFieldNames()) {
        if (!o.m_doc.containsKey(fieldName)) {
          return false; // a key was found in src but not in the dest
        }
        final Object srcValue = m_doc.get(fieldName);
        final Object destValue = o.m_doc.get(fieldName);

        if (srcValue == null ^ destValue == null) {
          return false; // one value is null
        } else if (srcValue != null && destValue != null) {
          if (srcValue instanceof Collection && destValue instanceof Collection) {
            Collection srcThings = (Collection) srcValue;
            Collection destThings = (Collection) destValue;

            if (srcThings.size() != destThings.size()) {
              return false; // src and dest contain different counts of elements
            }
            if (!srcThings.containsAll(destThings)) {
              return false; // contents of src and dest do not match
            }
          } else if (!srcValue.equals(destValue)) {
            return false; // values are different
          }
        } // else { both null and equal }
      }
    } else if ((m_doc != null) ^ (o.m_doc != null)) {
      return false; // one has a document and the other doesn't
    }
    // else { both are null and equal }
    return true;
  }
Esempio n. 14
0
 public static final SolrDocument toSolrDocument(Document doc, final IndexSchema schema) {
   SolrDocument out = new SolrDocument();
   for (IndexableField f : doc.getFields()) {
     // Make sure multivalued fields are represented as lists
     Object existing = out.get(f.name());
     if (existing == null) {
       SchemaField sf = schema.getFieldOrNull(f.name());
       if (sf != null && sf.multiValued()) {
         List<Object> vals = new ArrayList<>();
         vals.add(f);
         out.setField(f.name(), vals);
       } else {
         out.setField(f.name(), f);
       }
     } else {
       out.addField(f.name(), f);
     }
   }
   return out;
 }
Esempio n. 15
0
  @Test
  public void testGetId() throws Exception {
    SolrDocument rsp = getSolrClient().getById("0");
    assertNull(rsp);

    rsp = getSolrClient().getById("1");
    assertEquals("1", rsp.get("id"));
    assertEquals("Microsoft", rsp.get("term_s"));
    assertEquals("MSFT", rsp.get("term2_s"));

    rsp = getSolrClient().getById("2");
    assertEquals("2", rsp.get("id"));
    assertEquals("Apple", rsp.get("term_s"));
    assertEquals("AAPL", rsp.get("term2_s"));
  }
Esempio n. 16
0
  @Test
  public void testGetIdWithParams() throws Exception {
    final SolrParams ID_FL_ONLY = params(CommonParams.FL, "id");

    SolrDocument rsp = getSolrClient().getById("0", ID_FL_ONLY);
    assertNull(rsp);

    rsp = getSolrClient().getById("1", ID_FL_ONLY);
    assertEquals("1", rsp.get("id"));
    assertNull("This field should have been removed from the response.", rsp.get("term_s"));
    assertNull("This field should have been removed from the response.", rsp.get("term2_s"));

    rsp = getSolrClient().getById("2", ID_FL_ONLY);
    assertEquals("2", rsp.get("id"));
    assertNull("This field should have been removed from the response.", rsp.get("term_s"));
    assertNull("This field should have been removed from the response.", rsp.get("term2_s"));
  }
Esempio n. 17
0
  private static SolrDocument toSolrDoc(StoredDocument doc, IndexSchema schema) {
    SolrDocument out = new SolrDocument();
    for (StorableField f : doc.getFields()) {
      // Make sure multivalued fields are represented as lists
      Object existing = out.get(f.name());
      if (existing == null) {
        SchemaField sf = schema.getFieldOrNull(f.name());

        // don't return copyField targets
        if (sf != null && schema.isCopyFieldTarget(sf)) continue;

        if (sf != null && sf.multiValued()) {
          List<Object> vals = new ArrayList<Object>();
          vals.add(f);
          out.setField(f.name(), vals);
        } else {
          out.setField(f.name(), f);
        }
      } else {
        out.addField(f.name(), f);
      }
    }
    return out;
  }
  private JSONObject runSearch(HashMap<String, JSONObject> currMap) throws ServletException {
    JSONObject qJson = new JSONObject();

    try {

      for (Map.Entry<String, JSONObject> entry : currMap.entrySet()) {
        JSONObject resultAllJson = new JSONObject();
        String key = entry.getKey();
        JSONObject json = (JSONObject) entry.getValue();
        String queryVal = json.getString("query");

        // System.out.println("query: " + json.toString());

        // continue with properties list
        String searchType = null;
        int limit = 10; // default
        String typeStrict = "should"; // default
        ArrayList<String[]> propertiesList = new ArrayList<String[]>();

        if (json.has("type")) {
          searchType = json.getString("type");
        }
        if (json.has("limit")) {
          limit = json.getInt("limit");
        }
        if (json.has("type_strict")) { // Not sure what this variable
          // represents. Skip for now.
          typeStrict = json.getString("type_strict");
        }
        if (json.has("properties")) {
          JSONArray properties = json.getJSONArray("properties");
          for (int i = 0; i < properties.length(); i++) {
            String[] pvPair = new String[2];
            JSONObject jsonProperty = properties.getJSONObject(i);
            String pid = jsonProperty.getString("pid");
            String v = jsonProperty.getString("v");
            if (pid != null && v != null) {
              pvPair[0] = pid;
              pvPair[1] = v;
              propertiesList.add(pvPair);
            }
          }
        }

        // begin search
        JSONArray resultJsonArr = new JSONArray();

        // Solr
        SolrQuery query = getQuery(queryVal, searchType, limit, propertiesList);
        QueryResponse queryResponse = null;
        if (query != null) {
          SolrServer solr = SolrSetup.getSolrServer(getServletContext());
          queryResponse = solr.query(query);
        } else {
          log.error("Query for a search was null");
        }

        SolrDocumentList docs = null;
        if (queryResponse != null) {
          docs = queryResponse.getResults();
        } else {
          log.error("Query response for a search was null");
        }

        if (docs != null) {

          List<SearchResult> results = new ArrayList<SearchResult>();
          for (SolrDocument doc : docs) {
            try {
              String uri = doc.get(VitroSearchTermNames.URI).toString();
              // RY 7/1/2011
              // Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get()
              // returns a list.
              // Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get()
              // could return a list
              // But in fact: I'm no longer seeing any lists returned for individuals with multiple
              // labels. Not sure
              // if this is new behavior or what. ???
              Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
              String name = null;
              if (nameRaw instanceof List<?>) {
                @SuppressWarnings("unchecked")
                List<String> nameRawList = (List<String>) nameRaw;
                name = nameRawList.get(0);
              } else {
                name = (String) nameRaw;
              }

              SearchResult result = new SearchResult(name, uri);

              // populate result for Google Refine
              JSONObject resultJson = new JSONObject();
              resultJson.put("score", doc.getFieldValue("score"));
              String modUri = result.getUri().replace("#", "%23");
              resultJson.put("id", modUri);
              resultJson.put("name", result.getLabel());

              Collection<Object> rdfTypes = doc.getFieldValues(VitroSearchTermNames.RDFTYPE);
              JSONArray typesJsonArr = new JSONArray();
              if (rdfTypes != null) {

                for (Object rdfType : rdfTypes) {

                  // e.g.
                  // http://aims.fao.org/aos/geopolitical.owl#area
                  String type = (String) rdfType;
                  int lastIndex2 = type.lastIndexOf('/') + 1;
                  String typeName = type.substring(lastIndex2);
                  typeName = typeName.replace("#", ":");
                  JSONObject typesJson = new JSONObject();
                  typesJson.put("id", type);
                  typesJson.put("name", typeName);
                  typesJsonArr.put(typesJson);
                }
              }

              resultJson.put("type", typesJsonArr);
              resultJson.put("match", "false");
              resultJsonArr.put(resultJson);

            } catch (Exception e) {
              log.error(
                  "problem getting usable individuals from search " + "hits" + e.getMessage());
            }
          }
        } else {
          log.error("Docs for a search was null");
        }
        resultAllJson.put("result", resultJsonArr);
        qJson.put(key, resultAllJson);
        // System.out.println("results: " + qJson.toString());
      }

    } catch (JSONException ex) {
      log.error("JSONException: " + ex);
      throw new ServletException("JSONReconcileServlet JSONException: " + ex);
    } catch (SolrServerException ex) {
      log.error("JSONException: " + ex);
      throw new ServletException("JSONReconcileServlet SolrServerException: " + ex);
    }

    return qJson;
  }
Esempio n. 19
0
 @Override
 public List<String> getProperty(String name) {
   return asStringList(doc.get(name));
 }
Esempio n. 20
0
 @Override
 public String getContext() {
   return (String) doc.get(SearchFields.CONTEXT_FIELD_NAME);
 }
Esempio n. 21
0
 @Override
 public String getResource() {
   return (String) doc.get(SearchFields.URI_FIELD_NAME);
 }
Esempio n. 22
0
 @Override
 public String getId() {
   return (String) doc.get(SearchFields.ID_FIELD_NAME);
 }
Esempio n. 23
0
 public static Object getFieldValue(SolrDocument doc, String fieldName) {
   return (doc != null && doc.containsKey(fieldName)) ? doc.get(fieldName) : null;
 }
Esempio n. 24
0
  @SuppressWarnings("unchecked")
  public static void main(String a[]) throws Exception {
    // PrintWriter writer = new PrintWriter( "/users/mkchakravarti/flights/cds_greg" );
    SolrServer solrLocServer =
        new CommonsHttpSolrServer("http://indexedsearch.prod.o.com:8980/cds");
    // HashMap<String,String> locMap = getLocMap();
    SolrQuery solrQuery = new SolrQuery();
    solrQuery.addFilterQuery("status:publish", "contentType:TitleBodyMedialist");
    solrQuery.setQueryType("standard");
    for (int i = 0; i < 7; i++) {
      PrintWriter writer = new PrintWriter("/users/mkchakravarti/flights/cds_greg_ttl_" + i);
      solrQuery.setStart(i * 20000);
      solrQuery.setRows(20000);
      solrQuery.setFields(
          "urlType",
          "urlMapping",
          "contentPurpose",
          "contentType",
          "namedAttribute",
          "locale",
          "pos",
          "steerModuleKey");
      QueryResponse queryLocResponse = solrLocServer.query(solrQuery);
      System.out.println(solrQuery);
      SolrDocumentList locList = queryLocResponse.getResults();
      for (SolrDocument doc : locList) {
        try {
          String id = (String) doc.get("id");
          String contentPurpose = (String) doc.get("contentPurpose");

          ArrayList<String> steerModuleKey = (ArrayList<String>) doc.get("steerModuleKey");
          String contentType = (String) doc.get("contentType");
          ArrayList<String> obj = (ArrayList<String>) doc.get("namedAttribute");
          ArrayList<String> ut = (ArrayList<String>) doc.get("urlType");
          ArrayList<String> um = (ArrayList<String>) doc.get("urlMapping");
          ArrayList<String> pos = (ArrayList<String>) doc.get("pos");
          ArrayList<String> locale = (ArrayList<String>) doc.get("locale");
          String finalStr =
              id
                  + "_"
                  + contentPurpose
                  + "_"
                  + contentType
                  + "_"
                  + app(obj)
                  + "_"
                  + app(ut)
                  + "_"
                  + app(um)
                  + "_"
                  + app(pos)
                  + "_"
                  + app(locale)
                  + "_"
                  + app(steerModuleKey);
          writer.println(finalStr);

        } catch (Exception e) {
          System.out.println(e);
        }
      }
    }
  }