@Command
public void onAdd()
throws SearchLibException, TransformerConfigurationException, SAXException, IOException,
XPathExpressionException, ParserConfigurationException {
if (!isWebCrawlerParametersRights()) throw new SearchLibException("Not allowed");
if (selectedUrlField == null || selectedIndexField == null) return;
FieldMap fieldMap = getFieldMap();
fieldMap.add(
new SourceField(selectedUrlField.getName()), new TargetField(selectedIndexField.getName()));
fieldMap.store();
reload();
}
@Override
public void populate(List<IndexDocumentResult> indexDocuments)
throws IOException, SearchLibException {
SchemaFieldList schemaFieldList = request.getConfig().getSchema().getFieldList();
for (int docId : docArray) {
IndexDocumentResult indexDocument = new IndexDocumentResult(schemaFieldList.size());
Map<String, FieldValue> storedFieldMap = reader.getDocumentStoredField(docId);
for (SchemaField schemaField : schemaFieldList) {
String fieldName = schemaField.getName();
List<IndexTerm> indexTermList = null;
if (schemaField.checkIndexed(Indexed.YES)) {
if (schemaField.getTermVector() == TermVector.NO) {
indexTermList = IndexTerm.toList(reader, fieldName, docId);
} else {
TermFreqVector termFreqVector = reader.getTermFreqVector(docId, fieldName);
indexTermList = IndexTerm.toList(termFreqVector);
}
}
IndexField indexField =
new IndexField(fieldName, storedFieldMap.get(fieldName), indexTermList);
indexDocument.add(indexField);
}
indexDocuments.add(indexDocument);
}
}
/**
* Retourne la liste des champs "snippet".
*
* @param node
* @param source
* @param target
* @throws IllegalAccessException
* @throws InstantiationException
*/
public static void copySnippetFields(Node node, SchemaFieldList source, SnippetFieldList target)
throws InstantiationException, IllegalAccessException {
String fieldName = XPathParser.getAttributeString(node, "name");
String tag = XPathParser.getAttributeString(node, "tag");
if (tag == null) tag = "em";
int maxSnippetNumber = XPathParser.getAttributeValue(node, "maxSnippetNumber");
if (maxSnippetNumber == 0) maxSnippetNumber = 1;
int maxSnippetSize = XPathParser.getAttributeValue(node, "maxSnippetSize");
if (maxSnippetSize == 0) maxSnippetSize = 200;
int timeLimit = DomUtils.getAttributeInteger(node, "timeLimit", 0);
FragmenterAbstract fragmenter =
FragmenterAbstract.newInstance(XPathParser.getAttributeString(node, "fragmenterClass"));
fragmenter.setAttributes(node.getAttributes());
String separator = XPathParser.getAttributeString(node, "separator");
if (separator == null) separator = "...";
SchemaField schemaField = source.get(fieldName);
if (schemaField == null) return;
SnippetField field =
new SnippetField(
schemaField.getName(),
tag,
separator,
maxSnippetSize,
maxSnippetNumber,
fragmenter,
timeLimit);
target.put(field);
}
public List<String> getSnippetFieldLeft() throws SearchLibException {
synchronized (this) {
Client client = getClient();
if (client == null) return null;
AbstractSearchRequest request = (AbstractSearchRequest) getRequest();
if (request == null) return null;
if (snippetFieldLeft != null) return snippetFieldLeft;
snippetFieldLeft = new ArrayList<String>();
SnippetFieldList snippetFields = request.getSnippetFieldList();
for (SchemaField field : client.getSchema().getFieldList())
if (field.checkStored(Stored.YES, Stored.COMPRESS))
if (field.getTermVector() == TermVector.POSITIONS_OFFSETS)
if (snippetFields.get(field.getName()) == null) {
if (selectedSnippet == null) selectedSnippet = field.getName();
snippetFieldLeft.add(field.getName());
}
return snippetFieldLeft;
}
}
private static final int[] toDocArray(ReaderLocal reader, DocumentsRequest request)
throws IOException {
SchemaField schemaField = null;
Schema schema = request.getConfig().getSchema();
String field = request.getField();
if (!StringUtils.isEmpty(field)) {
schemaField = schema.getField(field);
if (schemaField == null) throw new IOException("Field not found: " + field);
} else {
schemaField = schema.getFieldList().getUniqueField();
if (schemaField == null) throw new IOException("No unique field");
}
int higher = -1;
RoaringBitmap bitSet = new RoaringBitmap();
String fieldName = schemaField.getName();
for (String uniqueKey : request.getUniqueKeyList()) {
TermDocs termDocs = reader.getTermDocs(new Term(fieldName, uniqueKey));
if (termDocs != null) {
while (termDocs.next()) {
int doc = termDocs.doc();
if (doc > higher) higher = doc;
bitSet.add(doc);
}
}
termDocs.close();
}
if (request.isReverse()) bitSet.flip(0, higher + 1);
IntBufferedArrayInterface intBufferArray =
IntBufferedArrayFactory.INSTANCE.newInstance(bitSet.getCardinality());
IntIterator iterator = bitSet.getIntIterator();
while (iterator.hasNext()) {
int docId = iterator.next();
if (!reader.isDeletedNoLock(docId)) intBufferArray.add(docId);
}
return intBufferArray.getFinalArray();
}