public void doTest(int[] docs) throws Exception { Directory dir = makeIndex(); IndexReader reader = IndexReader.open(dir, true); for (int i = 0; i < docs.length; i++) { Document d = reader.document(docs[i], SELECTOR); d.get(MAGIC_FIELD); List<Fieldable> fields = d.getFields(); for (Iterator<Fieldable> fi = fields.iterator(); fi.hasNext(); ) { Fieldable f = null; try { f = fi.next(); String fname = f.name(); String fval = f.stringValue(); assertNotNull(docs[i] + " FIELD: " + fname, fval); String[] vals = fval.split("#"); if (!dataset.contains(vals[0]) || !dataset.contains(vals[1])) { fail("FIELD:" + fname + ",VAL:" + fval); } } catch (Exception e) { throw new Exception(docs[i] + " WTF: " + f.name(), e); } } } reader.close(); dir.close(); }
private static SimpleOrderedMap<Object> getDocumentFieldsInfo( Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException { SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>(); for (Object o : doc.getFields()) { Fieldable fieldable = (Fieldable) o; SimpleOrderedMap<Object> f = new SimpleOrderedMap<Object>(); SchemaField sfield = schema.getFieldOrNull(fieldable.name()); FieldType ftype = (sfield == null) ? null : sfield.getType(); f.add("type", (ftype == null) ? null : ftype.getTypeName()); f.add("schema", getFieldFlags(sfield)); f.add("flags", getFieldFlags(fieldable)); Term t = new Term( fieldable.name(), ftype != null ? ftype.storedToIndexed(fieldable) : fieldable.stringValue()); f.add("value", (ftype == null) ? null : ftype.toExternal(fieldable)); // TODO: this really should be "stored" f.add("internal", fieldable.stringValue()); // may be a binary number byte[] arr = fieldable.getBinaryValue(); if (arr != null) { f.add("binary", Base64.byteArrayToBase64(arr, 0, arr.length)); } f.add("boost", fieldable.getBoost()); f.add( "docFreq", t.text() == null ? 0 : reader.docFreq(t)); // this can be 0 for non-indexed fields // If we have a term vector, return that if (fieldable.isTermVectorStored()) { try { TermFreqVector v = reader.getTermFreqVector(docId, fieldable.name()); if (v != null) { SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<Integer>(); for (int i = 0; i < v.size(); i++) { tfv.add(v.getTerms()[i], v.getTermFrequencies()[i]); } f.add("termVector", tfv); } } catch (Exception ex) { log.warn("error writing term vector", ex); } } finfo.add(fieldable.name(), f); } return finfo; }
private void dumpDocument(int docNum, Document doc) throws IOException { outputLn(); outputLn("Document " + docNum); if (doc == null) { outputLn(" deleted"); return; } // note: only stored fields will be returned for (Fieldable field : doc.getFields()) { String fieldName = field.name(); boolean isDate = "l.date".equals(fieldName); outputLn(" Field [" + fieldName + "]: " + field.toString()); String[] values = doc.getValues(fieldName); if (values != null) { int i = 0; for (String value : values) { output(" " + "(" + i++ + ") " + value); if (isDate) { try { Date date = DateTools.stringToDate(value); output(" (" + date.toString() + " (" + date.getTime() + "))"); } catch (java.text.ParseException e) { assert false; } } outputLn(); } } } }
@Override public int getOffsetGap(Fieldable field) { String fieldName = field.name(); int dotIndex = fieldName.indexOf('.'); if (dotIndex != -1) { String possibleType = fieldName.substring(0, dotIndex); DocumentMapper possibleDocMapper = mappers.get(possibleType); if (possibleDocMapper != null) { return possibleDocMapper.mappers().searchAnalyzer().getOffsetGap(field); } } FieldMappers mappers = fullNameFieldMappers.get(fieldName); if (mappers != null && mappers.mapper() != null && mappers.mapper().searchAnalyzer() != null) { return mappers.mapper().searchAnalyzer().getOffsetGap(field); } mappers = indexNameFieldMappers.get(fieldName); if (mappers != null && mappers.mapper() != null && mappers.mapper().searchAnalyzer() != null) { return mappers.mapper().searchAnalyzer().getOffsetGap(field); } return defaultAnalyzer.getOffsetGap(field); }
@Override public boolean reload(String collectionName, int docNum) { if (collectionName == null) return false; CrescentCollectionHandler collectionHandler = SpringApplicationContext.getBean( "crescentCollectionHandler", CrescentCollectionHandler.class); CrescentCollection collection = collectionHandler.getCrescentCollections().getCrescentCollection(collectionName); if (collection == null) { logger.debug("doesn't Collection Info => {}", collectionName); return false; } List<String> fieldName = new ArrayList<String>(); List<String> flag = new ArrayList<String>(); List<String> norm = new ArrayList<String>(); List<String> value = new ArrayList<String>(); try { Directory directory = FSDirectory.open(new File(collection.getIndexingDirectory())); IndexReader reader = IndexReader.open(directory); Document document = null; try { document = reader.document(docNum); } catch (IllegalArgumentException e) { e.printStackTrace(); return false; } String fName = null; for (Fieldable field : document.getFields()) { fName = field.name(); fieldName.add(fName); flag.add(fieldFlag(field)); if (reader.hasNorms(fName)) { norm.add(String.valueOf(Similarity.decodeNorm(reader.norms(fName)[docNum]))); } else { norm.add("---"); } value.add(field.stringValue()); } } catch (IOException e) { e.printStackTrace(); return false; } result.put("collection", collectionName); result.put("docNum", docNum); result.put("fieldName", fieldName); result.put("flag", flag); result.put("norm", norm); result.put("value", value); return true; }
@Override protected void onRecord(DiscoveredRecord record, Document document) { Map<String, List<String>> fieldMap = new HashMap<String, List<String>>(); for (Fieldable field : document.getFields()) { String name = field.name(); List<String> fieldValues = fieldMap.get(name); if (fieldValues == null) { fieldValues = new ArrayList<String>(); fieldMap.put(name, fieldValues); } fieldValues.add(field.stringValue()); } mapping.put(record, fieldMap); }
private static List<Fieldable> getNonEmptyFields(final Document doc) { @SuppressWarnings({"unchecked"}) final List<Fieldable> fields = doc.getFields(); final List<Fieldable> allVals = Lists.newArrayList(); for (final Fieldable field : fields) { // NOTE: we do not store the field value since we are never interested in reading the value // out of the // document, we are just interested in searching it. This will keep us from adding to the size // of the issue // document. if (field.isIndexed()) { allVals.add( new Field( DocumentConstants.ISSUE_NON_EMPTY_FIELD_IDS, field.name(), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS)); } } return allVals; }
public void writeDoc( String name, Collection<Fieldable> fields, Set<String> returnFields, Map pseudoFields) throws IOException { writeMapOpener(-1); // no trivial way to determine map size incLevel(); HashMap<String, MultiValueField> multi = new HashMap<String, MultiValueField>(); boolean first = true; for (Fieldable ff : fields) { String fname = ff.name(); if (returnFields != null && !returnFields.contains(fname)) { continue; } // if the field is multivalued, it may have other values further on... so // build up a list for each multi-valued field. SchemaField sf = schema.getField(fname); if (sf.multiValued()) { MultiValueField mf = multi.get(fname); if (mf == null) { mf = new MultiValueField(sf, ff); multi.put(fname, mf); } else { mf.fields.add(ff); } } else { // not multi-valued, so write it immediately. if (first) { first = false; } else { writeMapSeparator(); } indent(); writeKey(fname, true); sf.write(this, fname, ff); } } for (MultiValueField mvf : multi.values()) { if (first) { first = false; } else { writeMapSeparator(); } indent(); writeKey(mvf.sfield.getName(), true); boolean indentArrElems = false; if (doIndent) { // heuristic... TextField is probably the only field type likely to be long enough // to warrant indenting individual values. indentArrElems = (mvf.sfield.getType() instanceof TextField); } writeArrayOpener(-1); // no trivial way to determine array size boolean firstArrElem = true; incLevel(); for (Fieldable ff : mvf.fields) { if (firstArrElem) { firstArrElem = false; } else { writeArraySeparator(); } if (indentArrElems) indent(); mvf.sfield.write(this, null, ff); } writeArrayCloser(); decLevel(); } if (pseudoFields != null && pseudoFields.size() > 0) { writeMap(null, pseudoFields, true, first); } decLevel(); writeMapCloser(); }
public void parse(ParseContext context) throws IOException { if (!enabled) { context.parser().skipChildren(); return; } XContentParser parser = context.parser(); String currentFieldName = parser.currentName(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_NULL) { // the object is null ("obj1" : null), simply bail return; } Document restoreDoc = null; if (nested.isNested()) { Document nestedDoc = new Document(); // pre add the uid field if possible (id was already provided) Fieldable uidField = context.doc().getFieldable(UidFieldMapper.NAME); if (uidField != null) { // we don't need to add it as a full uid field in nested docs, since we don't need // versioning // we also rely on this for UidField#loadVersion // this is a deeply nested field if (uidField.stringValue() != null) { nestedDoc.add( new Field( UidFieldMapper.NAME, uidField.stringValue(), Field.Store.NO, Field.Index.NOT_ANALYZED)); } else { nestedDoc.add( new Field( UidFieldMapper.NAME, ((UidField) uidField).uid(), Field.Store.NO, Field.Index.NOT_ANALYZED)); } } // the type of the nested doc starts with __, so we can identify that its a nested one in // filters // note, we don't prefix it with the type of the doc since it allows us to execute a nested // query // across types (for example, with similar nested objects) nestedDoc.add( new Field( TypeFieldMapper.NAME, nestedTypePath, Field.Store.NO, Field.Index.NOT_ANALYZED)); restoreDoc = context.switchDoc(nestedDoc); context.addDoc(nestedDoc); } ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); // if we are at the end of the previous object, advance if (token == XContentParser.Token.END_OBJECT) { token = parser.nextToken(); } if (token == XContentParser.Token.START_OBJECT) { // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the // name first token = parser.nextToken(); } while (token != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.START_OBJECT) { serializeObject(context, currentFieldName); } else if (token == XContentParser.Token.START_ARRAY) { serializeArray(context, currentFieldName); } else if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, currentFieldName); } else if (token == null) { throw new MapperParsingException( "object mapping for [" + name + "] tried to parse as object, but got EOF, has a concrete value been provided to it?"); } else if (token.isValue()) { serializeValue(context, currentFieldName, token); } token = parser.nextToken(); } // restore the enable path flag context.path().pathType(origPathType); if (nested.isNested()) { Document nestedDoc = context.switchDoc(restoreDoc); if (nested.isIncludeInParent()) { for (Fieldable field : nestedDoc.getFields()) { if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { continue; } else { context.doc().add(field); } } } if (nested.isIncludeInRoot()) { // don't add it twice, if its included in parent, and we are handling the master doc... if (!(nested.isIncludeInParent() && context.doc() == context.rootDoc())) { for (Fieldable field : nestedDoc.getFields()) { if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { continue; } else { context.rootDoc().add(field); } } } } } }
@Override protected GetResponse shardOperation(GetRequest request, int shardId) throws ElasticSearchException { IndexService indexService = indicesService.indexServiceSafe(request.index()); BloomCache bloomCache = indexService.cache().bloomCache(); IndexShard indexShard = indexService.shardSafe(shardId); DocumentMapper docMapper = indexService.mapperService().documentMapper(request.type()); if (docMapper == null) { throw new TypeMissingException(new Index(request.index()), request.type()); } if (request.refresh()) { indexShard.refresh(new Engine.Refresh(false)); } Engine.Searcher searcher = indexShard.searcher(); boolean exists = false; byte[] source = null; Map<String, GetField> fields = null; long version = -1; try { UidField.DocIdAndVersion docIdAndVersion = loadCurrentVersionFromIndex( bloomCache, searcher, docMapper.uidMapper().term(request.type(), request.id())); if (docIdAndVersion != null && docIdAndVersion.docId != Lucene.NO_DOC) { if (docIdAndVersion.version > 0) { version = docIdAndVersion.version; } exists = true; FieldSelector fieldSelector = buildFieldSelectors(docMapper, request.fields()); if (fieldSelector != null) { Document doc = docIdAndVersion.reader.document(docIdAndVersion.docId, fieldSelector); source = extractSource(doc, docMapper); for (Object oField : doc.getFields()) { Fieldable field = (Fieldable) oField; String name = field.name(); Object value = null; FieldMappers fieldMappers = docMapper.mappers().indexName(field.name()); if (fieldMappers != null) { FieldMapper mapper = fieldMappers.mapper(); if (mapper != null) { name = mapper.names().fullName(); value = mapper.valueForSearch(field); } } if (value == null) { if (field.isBinary()) { value = field.getBinaryValue(); } else { value = field.stringValue(); } } if (fields == null) { fields = newHashMapWithExpectedSize(2); } GetField getField = fields.get(name); if (getField == null) { getField = new GetField(name, new ArrayList<Object>(2)); fields.put(name, getField); } getField.values().add(value); } } // now, go and do the script thingy if needed if (request.fields() != null && request.fields().length > 0) { SearchLookup searchLookup = null; for (String field : request.fields()) { String script = null; if (field.contains("_source.") || field.contains("doc[")) { script = field; } else { FieldMappers x = docMapper.mappers().smartName(field); if (x != null && !x.mapper().stored()) { script = "_source." + x.mapper().names().fullName(); } } if (script != null) { if (searchLookup == null) { searchLookup = new SearchLookup( indexService.mapperService(), indexService.cache().fieldData()); } SearchScript searchScript = scriptService.search(searchLookup, "mvel", script, null); searchScript.setNextReader(docIdAndVersion.reader); searchScript.setNextDocId(docIdAndVersion.docId); try { Object value = searchScript.run(); if (fields == null) { fields = newHashMapWithExpectedSize(2); } GetField getField = fields.get(field); if (getField == null) { getField = new GetField(field, new ArrayList<Object>(2)); fields.put(field, getField); } getField.values().add(value); } catch (RuntimeException e) { if (logger.isTraceEnabled()) { logger.trace("failed to execute get request script field [{}]", e, script); } // ignore } } } } } } catch (IOException e) { throw new ElasticSearchException( "Failed to get type [" + request.type() + "] and id [" + request.id() + "]", e); } finally { searcher.release(); } return new GetResponse( request.index(), request.type(), request.id(), version, exists, source, fields); }
@Override public String storedToIndexed(Fieldable f) { if (f instanceof NumericField) { final Number val = ((NumericField) f).getNumericValue(); if (val == null) throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: " + f.name()); switch (type) { case INTEGER: return NumericUtils.intToPrefixCoded(val.intValue()); case FLOAT: return NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(val.floatValue())); case LONG: // fallthrough! case DATE: return NumericUtils.longToPrefixCoded(val.longValue()); case DOUBLE: return NumericUtils.longToPrefixCoded( NumericUtils.doubleToSortableLong(val.doubleValue())); default: throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } } else { // the following code is "deprecated" and only to support pre-3.2 indexes using the old // BinaryField encoding: final byte[] arr = f.getBinaryValue(); if (arr == null) throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: " + f.name()); switch (type) { case INTEGER: return NumericUtils.intToPrefixCoded(toInt(arr)); case FLOAT: { // WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils! // copied from NumericUtils to not convert to/from float two times // code in next 2 lines is identical to: int v = // NumericUtils.floatToSortableInt(Float.intBitsToFloat(toInt(arr))); int v = toInt(arr); if (v < 0) v ^= 0x7fffffff; return NumericUtils.intToPrefixCoded(v); } case LONG: // fallthrough! case DATE: return NumericUtils.longToPrefixCoded(toLong(arr)); case DOUBLE: { // WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils! // copied from NumericUtils to not convert to/from double two times // code in next 2 lines is identical to: long v = // NumericUtils.doubleToSortableLong(Double.longBitsToDouble(toLong(arr))); long v = toLong(arr); if (v < 0) v ^= 0x7fffffffffffffffL; return NumericUtils.longToPrefixCoded(v); } default: throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } } }
@Override public Object toObject(Fieldable f) { if (f instanceof NumericField) { final Number val = ((NumericField) f).getNumericValue(); if (val == null) return badFieldString(f); return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val; } else { // the following code is "deprecated" and only to support pre-3.2 indexes using the old // BinaryField encoding: final byte[] arr = f.getBinaryValue(); if (arr == null) return badFieldString(f); switch (type) { case INTEGER: return toInt(arr); case FLOAT: return Float.intBitsToFloat(toInt(arr)); case LONG: return toLong(arr); case DOUBLE: return Double.longBitsToDouble(toLong(arr)); case DATE: return new Date(toLong(arr)); default: throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } } }