@Override
 public void populate(List<IndexDocumentResult> indexDocuments)
     throws IOException, SearchLibException {
   SchemaFieldList schemaFieldList = request.getConfig().getSchema().getFieldList();
   for (int docId : docArray) {
     IndexDocumentResult indexDocument = new IndexDocumentResult(schemaFieldList.size());
     Map<String, FieldValue> storedFieldMap = reader.getDocumentStoredField(docId);
     for (SchemaField schemaField : schemaFieldList) {
       String fieldName = schemaField.getName();
       List<IndexTerm> indexTermList = null;
       if (schemaField.checkIndexed(Indexed.YES)) {
         if (schemaField.getTermVector() == TermVector.NO) {
           indexTermList = IndexTerm.toList(reader, fieldName, docId);
         } else {
           TermFreqVector termFreqVector = reader.getTermFreqVector(docId, fieldName);
           indexTermList = IndexTerm.toList(termFreqVector);
         }
       }
       IndexField indexField =
           new IndexField(fieldName, storedFieldMap.get(fieldName), indexTermList);
       indexDocument.add(indexField);
     }
     indexDocuments.add(indexDocument);
   }
 }
Ejemplo n.º 2
0
  /**
   * Retourne la liste des champs "snippet".
   *
   * @param node
   * @param source
   * @param target
   * @throws IllegalAccessException
   * @throws InstantiationException
   */
  public static void copySnippetFields(Node node, SchemaFieldList source, SnippetFieldList target)
      throws InstantiationException, IllegalAccessException {
    String fieldName = XPathParser.getAttributeString(node, "name");
    String tag = XPathParser.getAttributeString(node, "tag");
    if (tag == null) tag = "em";
    int maxSnippetNumber = XPathParser.getAttributeValue(node, "maxSnippetNumber");
    if (maxSnippetNumber == 0) maxSnippetNumber = 1;
    int maxSnippetSize = XPathParser.getAttributeValue(node, "maxSnippetSize");
    if (maxSnippetSize == 0) maxSnippetSize = 200;
    int timeLimit = DomUtils.getAttributeInteger(node, "timeLimit", 0);

    FragmenterAbstract fragmenter =
        FragmenterAbstract.newInstance(XPathParser.getAttributeString(node, "fragmenterClass"));
    fragmenter.setAttributes(node.getAttributes());
    String separator = XPathParser.getAttributeString(node, "separator");
    if (separator == null) separator = "...";
    SchemaField schemaField = source.get(fieldName);
    if (schemaField == null) return;
    SnippetField field =
        new SnippetField(
            schemaField.getName(),
            tag,
            separator,
            maxSnippetSize,
            maxSnippetNumber,
            fragmenter,
            timeLimit);
    target.put(field);
  }
Ejemplo n.º 3
0
 @Command
 public void onAdd()
     throws SearchLibException, TransformerConfigurationException, SAXException, IOException,
         XPathExpressionException, ParserConfigurationException {
   if (!isWebCrawlerParametersRights()) throw new SearchLibException("Not allowed");
   if (selectedUrlField == null || selectedIndexField == null) return;
   FieldMap fieldMap = getFieldMap();
   fieldMap.add(
       new SourceField(selectedUrlField.getName()), new TargetField(selectedIndexField.getName()));
   fieldMap.store();
   reload();
 }
 private static final int[] toDocArray(ReaderLocal reader, DocumentsRequest request)
     throws IOException {
   SchemaField schemaField = null;
   Schema schema = request.getConfig().getSchema();
   String field = request.getField();
   if (!StringUtils.isEmpty(field)) {
     schemaField = schema.getField(field);
     if (schemaField == null) throw new IOException("Field not found: " + field);
   } else {
     schemaField = schema.getFieldList().getUniqueField();
     if (schemaField == null) throw new IOException("No unique field");
   }
   int higher = -1;
   RoaringBitmap bitSet = new RoaringBitmap();
   String fieldName = schemaField.getName();
   for (String uniqueKey : request.getUniqueKeyList()) {
     TermDocs termDocs = reader.getTermDocs(new Term(fieldName, uniqueKey));
     if (termDocs != null) {
       while (termDocs.next()) {
         int doc = termDocs.doc();
         if (doc > higher) higher = doc;
         bitSet.add(doc);
       }
     }
     termDocs.close();
   }
   if (request.isReverse()) bitSet.flip(0, higher + 1);
   IntBufferedArrayInterface intBufferArray =
       IntBufferedArrayFactory.INSTANCE.newInstance(bitSet.getCardinality());
   IntIterator iterator = bitSet.getIntIterator();
   while (iterator.hasNext()) {
     int docId = iterator.next();
     if (!reader.isDeletedNoLock(docId)) intBufferArray.add(docId);
   }
   return intBufferArray.getFinalArray();
 }
Ejemplo n.º 5
0
 protected static final Facet facetMultivalued(
     ReaderAbstract reader,
     SchemaField schemaField,
     DocIdInterface docIdInterface,
     FacetField facetField,
     Timer timer)
     throws IOException, SearchLibException {
   String fieldName = facetField.getName();
   if (schemaField.getTermVector() == TermVector.NO) {
     FieldCacheIndex stringIndex = reader.getStringIndex(fieldName);
     int[] countIndex = computeMultivaluedTD(reader, fieldName, stringIndex, docIdInterface);
     return new Facet(facetField, stringIndex.lookup, countIndex);
   } else {
     Map<String, FacetItem> facetMap = computeMultivaluedTFV(reader, fieldName, docIdInterface);
     return new Facet(facetField, facetMap);
   }
 }
 public List<String> getSnippetFieldLeft() throws SearchLibException {
   synchronized (this) {
     Client client = getClient();
     if (client == null) return null;
     AbstractSearchRequest request = (AbstractSearchRequest) getRequest();
     if (request == null) return null;
     if (snippetFieldLeft != null) return snippetFieldLeft;
     snippetFieldLeft = new ArrayList<String>();
     SnippetFieldList snippetFields = request.getSnippetFieldList();
     for (SchemaField field : client.getSchema().getFieldList())
       if (field.checkStored(Stored.YES, Stored.COMPRESS))
         if (field.getTermVector() == TermVector.POSITIONS_OFFSETS)
           if (snippetFields.get(field.getName()) == null) {
             if (selectedSnippet == null) selectedSnippet = field.getName();
             snippetFieldLeft.add(field.getName());
           }
     return snippetFieldLeft;
   }
 }