public void doTest(int[] docs) throws Exception {
Directory dir = makeIndex();
IndexReader reader = IndexReader.open(dir, true);
for (int i = 0; i < docs.length; i++) {
Document d = reader.document(docs[i], SELECTOR);
d.get(MAGIC_FIELD);
List<Fieldable> fields = d.getFields();
for (Iterator<Fieldable> fi = fields.iterator(); fi.hasNext(); ) {
Fieldable f = null;
try {
f = fi.next();
String fname = f.name();
String fval = f.stringValue();
assertNotNull(docs[i] + " FIELD: " + fname, fval);
String[] vals = fval.split("#");
if (!dataset.contains(vals[0]) || !dataset.contains(vals[1])) {
fail("FIELD:" + fname + ",VAL:" + fval);
}
} catch (Exception e) {
throw new Exception(docs[i] + " WTF: " + f.name(), e);
}
}
}
reader.close();
dir.close();
}
private static SimpleOrderedMap<Object> getDocumentFieldsInfo(
Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException {
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
for (Object o : doc.getFields()) {
Fieldable fieldable = (Fieldable) o;
SimpleOrderedMap<Object> f = new SimpleOrderedMap<Object>();
SchemaField sfield = schema.getFieldOrNull(fieldable.name());
FieldType ftype = (sfield == null) ? null : sfield.getType();
f.add("type", (ftype == null) ? null : ftype.getTypeName());
f.add("schema", getFieldFlags(sfield));
f.add("flags", getFieldFlags(fieldable));
Term t =
new Term(
fieldable.name(),
ftype != null ? ftype.storedToIndexed(fieldable) : fieldable.stringValue());
f.add("value", (ftype == null) ? null : ftype.toExternal(fieldable));
// TODO: this really should be "stored"
f.add("internal", fieldable.stringValue()); // may be a binary number
byte[] arr = fieldable.getBinaryValue();
if (arr != null) {
f.add("binary", Base64.byteArrayToBase64(arr, 0, arr.length));
}
f.add("boost", fieldable.getBoost());
f.add(
"docFreq",
t.text() == null ? 0 : reader.docFreq(t)); // this can be 0 for non-indexed fields
// If we have a term vector, return that
if (fieldable.isTermVectorStored()) {
try {
TermFreqVector v = reader.getTermFreqVector(docId, fieldable.name());
if (v != null) {
SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<Integer>();
for (int i = 0; i < v.size(); i++) {
tfv.add(v.getTerms()[i], v.getTermFrequencies()[i]);
}
f.add("termVector", tfv);
}
} catch (Exception ex) {
log.warn("error writing term vector", ex);
}
}
finfo.add(fieldable.name(), f);
}
return finfo;
}
private void dumpDocument(int docNum, Document doc) throws IOException {
outputLn();
outputLn("Document " + docNum);
if (doc == null) {
outputLn(" deleted");
return;
}
// note: only stored fields will be returned
for (Fieldable field : doc.getFields()) {
String fieldName = field.name();
boolean isDate = "l.date".equals(fieldName);
outputLn(" Field [" + fieldName + "]: " + field.toString());
String[] values = doc.getValues(fieldName);
if (values != null) {
int i = 0;
for (String value : values) {
output(" " + "(" + i++ + ") " + value);
if (isDate) {
try {
Date date = DateTools.stringToDate(value);
output(" (" + date.toString() + " (" + date.getTime() + "))");
} catch (java.text.ParseException e) {
assert false;
}
}
outputLn();
}
}
}
}
@Override
public int getOffsetGap(Fieldable field) {
String fieldName = field.name();
int dotIndex = fieldName.indexOf('.');
if (dotIndex != -1) {
String possibleType = fieldName.substring(0, dotIndex);
DocumentMapper possibleDocMapper = mappers.get(possibleType);
if (possibleDocMapper != null) {
return possibleDocMapper.mappers().searchAnalyzer().getOffsetGap(field);
}
}
FieldMappers mappers = fullNameFieldMappers.get(fieldName);
if (mappers != null
&& mappers.mapper() != null
&& mappers.mapper().searchAnalyzer() != null) {
return mappers.mapper().searchAnalyzer().getOffsetGap(field);
}
mappers = indexNameFieldMappers.get(fieldName);
if (mappers != null
&& mappers.mapper() != null
&& mappers.mapper().searchAnalyzer() != null) {
return mappers.mapper().searchAnalyzer().getOffsetGap(field);
}
return defaultAnalyzer.getOffsetGap(field);
}
@Override
public boolean reload(String collectionName, int docNum) {
if (collectionName == null) return false;
CrescentCollectionHandler collectionHandler =
SpringApplicationContext.getBean(
"crescentCollectionHandler", CrescentCollectionHandler.class);
CrescentCollection collection =
collectionHandler.getCrescentCollections().getCrescentCollection(collectionName);
if (collection == null) {
logger.debug("doesn't Collection Info => {}", collectionName);
return false;
}
List<String> fieldName = new ArrayList<String>();
List<String> flag = new ArrayList<String>();
List<String> norm = new ArrayList<String>();
List<String> value = new ArrayList<String>();
try {
Directory directory = FSDirectory.open(new File(collection.getIndexingDirectory()));
IndexReader reader = IndexReader.open(directory);
Document document = null;
try {
document = reader.document(docNum);
} catch (IllegalArgumentException e) {
e.printStackTrace();
return false;
}
String fName = null;
for (Fieldable field : document.getFields()) {
fName = field.name();
fieldName.add(fName);
flag.add(fieldFlag(field));
if (reader.hasNorms(fName)) {
norm.add(String.valueOf(Similarity.decodeNorm(reader.norms(fName)[docNum])));
} else {
norm.add("---");
}
value.add(field.stringValue());
}
} catch (IOException e) {
e.printStackTrace();
return false;
}
result.put("collection", collectionName);
result.put("docNum", docNum);
result.put("fieldName", fieldName);
result.put("flag", flag);
result.put("norm", norm);
result.put("value", value);
return true;
}
@Override
protected void onRecord(DiscoveredRecord record, Document document) {
Map<String, List<String>> fieldMap = new HashMap<String, List<String>>();
for (Fieldable field : document.getFields()) {
String name = field.name();
List<String> fieldValues = fieldMap.get(name);
if (fieldValues == null) {
fieldValues = new ArrayList<String>();
fieldMap.put(name, fieldValues);
}
fieldValues.add(field.stringValue());
}
mapping.put(record, fieldMap);
}
private static List<Fieldable> getNonEmptyFields(final Document doc) {
@SuppressWarnings({"unchecked"})
final List<Fieldable> fields = doc.getFields();
final List<Fieldable> allVals = Lists.newArrayList();
for (final Fieldable field : fields) {
// NOTE: we do not store the field value since we are never interested in reading the value
// out of the
// document, we are just interested in searching it. This will keep us from adding to the size
// of the issue
// document.
if (field.isIndexed()) {
allVals.add(
new Field(
DocumentConstants.ISSUE_NON_EMPTY_FIELD_IDS,
field.name(),
Field.Store.NO,
Field.Index.NOT_ANALYZED_NO_NORMS));
}
}
return allVals;
}
public void writeDoc(
String name, Collection<Fieldable> fields, Set<String> returnFields, Map pseudoFields)
throws IOException {
writeMapOpener(-1); // no trivial way to determine map size
incLevel();
HashMap<String, MultiValueField> multi = new HashMap<String, MultiValueField>();
boolean first = true;
for (Fieldable ff : fields) {
String fname = ff.name();
if (returnFields != null && !returnFields.contains(fname)) {
continue;
}
// if the field is multivalued, it may have other values further on... so
// build up a list for each multi-valued field.
SchemaField sf = schema.getField(fname);
if (sf.multiValued()) {
MultiValueField mf = multi.get(fname);
if (mf == null) {
mf = new MultiValueField(sf, ff);
multi.put(fname, mf);
} else {
mf.fields.add(ff);
}
} else {
// not multi-valued, so write it immediately.
if (first) {
first = false;
} else {
writeMapSeparator();
}
indent();
writeKey(fname, true);
sf.write(this, fname, ff);
}
}
for (MultiValueField mvf : multi.values()) {
if (first) {
first = false;
} else {
writeMapSeparator();
}
indent();
writeKey(mvf.sfield.getName(), true);
boolean indentArrElems = false;
if (doIndent) {
// heuristic... TextField is probably the only field type likely to be long enough
// to warrant indenting individual values.
indentArrElems = (mvf.sfield.getType() instanceof TextField);
}
writeArrayOpener(-1); // no trivial way to determine array size
boolean firstArrElem = true;
incLevel();
for (Fieldable ff : mvf.fields) {
if (firstArrElem) {
firstArrElem = false;
} else {
writeArraySeparator();
}
if (indentArrElems) indent();
mvf.sfield.write(this, null, ff);
}
writeArrayCloser();
decLevel();
}
if (pseudoFields != null && pseudoFields.size() > 0) {
writeMap(null, pseudoFields, true, first);
}
decLevel();
writeMapCloser();
}
public void parse(ParseContext context) throws IOException {
if (!enabled) {
context.parser().skipChildren();
return;
}
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
return;
}
Document restoreDoc = null;
if (nested.isNested()) {
Document nestedDoc = new Document();
// pre add the uid field if possible (id was already provided)
Fieldable uidField = context.doc().getFieldable(UidFieldMapper.NAME);
if (uidField != null) {
// we don't need to add it as a full uid field in nested docs, since we don't need
// versioning
// we also rely on this for UidField#loadVersion
// this is a deeply nested field
if (uidField.stringValue() != null) {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
uidField.stringValue(),
Field.Store.NO,
Field.Index.NOT_ANALYZED));
} else {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
((UidField) uidField).uid(),
Field.Store.NO,
Field.Index.NOT_ANALYZED));
}
}
// the type of the nested doc starts with __, so we can identify that its a nested one in
// filters
// note, we don't prefix it with the type of the doc since it allows us to execute a nested
// query
// across types (for example, with similar nested objects)
nestedDoc.add(
new Field(
TypeFieldMapper.NAME, nestedTypePath, Field.Store.NO, Field.Index.NOT_ANALYZED));
restoreDoc = context.switchDoc(nestedDoc);
context.addDoc(nestedDoc);
}
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
// if we are at the end of the previous object, advance
if (token == XContentParser.Token.END_OBJECT) {
token = parser.nextToken();
}
if (token == XContentParser.Token.START_OBJECT) {
// if we are just starting an OBJECT, advance, this is the object we are parsing, we need the
// name first
token = parser.nextToken();
}
while (token != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, currentFieldName);
} else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, currentFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, currentFieldName);
} else if (token == null) {
throw new MapperParsingException(
"object mapping for ["
+ name
+ "] tried to parse as object, but got EOF, has a concrete value been provided to it?");
} else if (token.isValue()) {
serializeValue(context, currentFieldName, token);
}
token = parser.nextToken();
}
// restore the enable path flag
context.path().pathType(origPathType);
if (nested.isNested()) {
Document nestedDoc = context.switchDoc(restoreDoc);
if (nested.isIncludeInParent()) {
for (Fieldable field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.doc().add(field);
}
}
}
if (nested.isIncludeInRoot()) {
// don't add it twice, if its included in parent, and we are handling the master doc...
if (!(nested.isIncludeInParent() && context.doc() == context.rootDoc())) {
for (Fieldable field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.rootDoc().add(field);
}
}
}
}
}
}
@Override
protected GetResponse shardOperation(GetRequest request, int shardId)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
BloomCache bloomCache = indexService.cache().bloomCache();
IndexShard indexShard = indexService.shardSafe(shardId);
DocumentMapper docMapper = indexService.mapperService().documentMapper(request.type());
if (docMapper == null) {
throw new TypeMissingException(new Index(request.index()), request.type());
}
if (request.refresh()) {
indexShard.refresh(new Engine.Refresh(false));
}
Engine.Searcher searcher = indexShard.searcher();
boolean exists = false;
byte[] source = null;
Map<String, GetField> fields = null;
long version = -1;
try {
UidField.DocIdAndVersion docIdAndVersion =
loadCurrentVersionFromIndex(
bloomCache, searcher, docMapper.uidMapper().term(request.type(), request.id()));
if (docIdAndVersion != null && docIdAndVersion.docId != Lucene.NO_DOC) {
if (docIdAndVersion.version > 0) {
version = docIdAndVersion.version;
}
exists = true;
FieldSelector fieldSelector = buildFieldSelectors(docMapper, request.fields());
if (fieldSelector != null) {
Document doc = docIdAndVersion.reader.document(docIdAndVersion.docId, fieldSelector);
source = extractSource(doc, docMapper);
for (Object oField : doc.getFields()) {
Fieldable field = (Fieldable) oField;
String name = field.name();
Object value = null;
FieldMappers fieldMappers = docMapper.mappers().indexName(field.name());
if (fieldMappers != null) {
FieldMapper mapper = fieldMappers.mapper();
if (mapper != null) {
name = mapper.names().fullName();
value = mapper.valueForSearch(field);
}
}
if (value == null) {
if (field.isBinary()) {
value = field.getBinaryValue();
} else {
value = field.stringValue();
}
}
if (fields == null) {
fields = newHashMapWithExpectedSize(2);
}
GetField getField = fields.get(name);
if (getField == null) {
getField = new GetField(name, new ArrayList<Object>(2));
fields.put(name, getField);
}
getField.values().add(value);
}
}
// now, go and do the script thingy if needed
if (request.fields() != null && request.fields().length > 0) {
SearchLookup searchLookup = null;
for (String field : request.fields()) {
String script = null;
if (field.contains("_source.") || field.contains("doc[")) {
script = field;
} else {
FieldMappers x = docMapper.mappers().smartName(field);
if (x != null && !x.mapper().stored()) {
script = "_source." + x.mapper().names().fullName();
}
}
if (script != null) {
if (searchLookup == null) {
searchLookup =
new SearchLookup(
indexService.mapperService(), indexService.cache().fieldData());
}
SearchScript searchScript = scriptService.search(searchLookup, "mvel", script, null);
searchScript.setNextReader(docIdAndVersion.reader);
searchScript.setNextDocId(docIdAndVersion.docId);
try {
Object value = searchScript.run();
if (fields == null) {
fields = newHashMapWithExpectedSize(2);
}
GetField getField = fields.get(field);
if (getField == null) {
getField = new GetField(field, new ArrayList<Object>(2));
fields.put(field, getField);
}
getField.values().add(value);
} catch (RuntimeException e) {
if (logger.isTraceEnabled()) {
logger.trace("failed to execute get request script field [{}]", e, script);
}
// ignore
}
}
}
}
}
} catch (IOException e) {
throw new ElasticSearchException(
"Failed to get type [" + request.type() + "] and id [" + request.id() + "]", e);
} finally {
searcher.release();
}
return new GetResponse(
request.index(), request.type(), request.id(), version, exists, source, fields);
}
@Override
public String storedToIndexed(Fieldable f) {
if (f instanceof NumericField) {
final Number val = ((NumericField) f).getNumericValue();
if (val == null)
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: " + f.name());
switch (type) {
case INTEGER:
return NumericUtils.intToPrefixCoded(val.intValue());
case FLOAT:
return NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(val.floatValue()));
case LONG: // fallthrough!
case DATE:
return NumericUtils.longToPrefixCoded(val.longValue());
case DOUBLE:
return NumericUtils.longToPrefixCoded(
NumericUtils.doubleToSortableLong(val.doubleValue()));
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
} else {
// the following code is "deprecated" and only to support pre-3.2 indexes using the old
// BinaryField encoding:
final byte[] arr = f.getBinaryValue();
if (arr == null)
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: " + f.name());
switch (type) {
case INTEGER:
return NumericUtils.intToPrefixCoded(toInt(arr));
case FLOAT:
{
// WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
// copied from NumericUtils to not convert to/from float two times
// code in next 2 lines is identical to: int v =
// NumericUtils.floatToSortableInt(Float.intBitsToFloat(toInt(arr)));
int v = toInt(arr);
if (v < 0) v ^= 0x7fffffff;
return NumericUtils.intToPrefixCoded(v);
}
case LONG: // fallthrough!
case DATE:
return NumericUtils.longToPrefixCoded(toLong(arr));
case DOUBLE:
{
// WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
// copied from NumericUtils to not convert to/from double two times
// code in next 2 lines is identical to: long v =
// NumericUtils.doubleToSortableLong(Double.longBitsToDouble(toLong(arr)));
long v = toLong(arr);
if (v < 0) v ^= 0x7fffffffffffffffL;
return NumericUtils.longToPrefixCoded(v);
}
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
}
}
@Override
public Object toObject(Fieldable f) {
if (f instanceof NumericField) {
final Number val = ((NumericField) f).getNumericValue();
if (val == null) return badFieldString(f);
return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val;
} else {
// the following code is "deprecated" and only to support pre-3.2 indexes using the old
// BinaryField encoding:
final byte[] arr = f.getBinaryValue();
if (arr == null) return badFieldString(f);
switch (type) {
case INTEGER:
return toInt(arr);
case FLOAT:
return Float.intBitsToFloat(toInt(arr));
case LONG:
return toLong(arr);
case DOUBLE:
return Double.longBitsToDouble(toLong(arr));
case DATE:
return new Date(toLong(arr));
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
}
}
