private void addField(
Document doc, FieldInfo fi, boolean binary, boolean compressed, boolean tokenize)
throws IOException {
// we have a binary stored field, and it may be compressed
if (binary) {
int toRead = fieldsStream.readVInt();
final byte[] b = new byte[toRead];
fieldsStream.readBytes(b, 0, b.length);
if (compressed) doc.add(new Field(fi.name, uncompress(b), Field.Store.COMPRESS));
else doc.add(new Field(fi.name, b, Field.Store.YES));
} else {
Field.Store store = Field.Store.YES;
Field.Index index = getIndexType(fi, tokenize);
Field.TermVector termVector = getTermVectorType(fi);
Fieldable f;
if (compressed) {
store = Field.Store.COMPRESS;
int toRead = fieldsStream.readVInt();
final byte[] b = new byte[toRead];
fieldsStream.readBytes(b, 0, b.length);
f =
new Field(
fi.name, // field name
new String(uncompress(b), "UTF-8"), // uncompress the value and add as string
store,
index,
termVector);
f.setOmitNorms(fi.omitNorms);
} else {
f =
new Field(
fi.name, // name
fieldsStream.readString(), // read value
store,
index,
termVector);
f.setOmitNorms(fi.omitNorms);
}
doc.add(f);
}
}
private void addFieldLazy(
Document doc, FieldInfo fi, boolean binary, boolean compressed, boolean tokenize)
throws IOException {
if (binary == true) {
int toRead = fieldsStream.readVInt();
long pointer = fieldsStream.getFilePointer();
if (compressed) {
// was: doc.add(new Fieldable(fi.name, uncompress(b), Fieldable.Store.COMPRESS));
doc.add(new LazyField(fi.name, Field.Store.COMPRESS, toRead, pointer));
} else {
// was: doc.add(new Fieldable(fi.name, b, Fieldable.Store.YES));
doc.add(new LazyField(fi.name, Field.Store.YES, toRead, pointer));
}
// Need to move the pointer ahead by toRead positions
fieldsStream.seek(pointer + toRead);
} else {
Field.Store store = Field.Store.YES;
Field.Index index = getIndexType(fi, tokenize);
Field.TermVector termVector = getTermVectorType(fi);
Fieldable f;
if (compressed) {
store = Field.Store.COMPRESS;
int toRead = fieldsStream.readVInt();
long pointer = fieldsStream.getFilePointer();
f = new LazyField(fi.name, store, toRead, pointer);
// skip over the part that we aren't loading
fieldsStream.seek(pointer + toRead);
f.setOmitNorms(fi.omitNorms);
} else {
int length = fieldsStream.readVInt();
long pointer = fieldsStream.getFilePointer();
// Skip ahead of where we are by the length of what is stored
fieldsStream.skipChars(length);
f = new LazyField(fi.name, store, index, termVector, length, pointer);
f.setOmitNorms(fi.omitNorms);
}
doc.add(f);
}
}
/*
* (non-Javadoc)
*
* @see org.apache.solr.schema.FieldType#createField(org.apache.solr.schema.SchemaField, java.lang.String, float)
*/
@Override
public Fieldable createField(SchemaField field, String externalVal, float boost) {
if (!field.indexed() && !field.stored()) {
if (log.isTraceEnabled()) log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
String val = null;
try {
val = toInternal(externalVal);
} catch (RuntimeException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Error while creating field '" + field + "' from value '" + externalVal + "'",
e,
false);
}
if (val == null) return null;
Fieldable f;
if (val.length() > compressionThreshold) {
f =
new CompressedField(
field.getName(),
val,
getFieldStore(field, val),
getFieldIndex(field, val),
getFieldTermVec(field, val),
compressionLevel);
} else {
f =
new Field(
field.getName(),
val,
getFieldStore(field, val),
getFieldIndex(field, val),
getFieldTermVec(field, val));
}
f.setOmitNorms(field.omitNorms());
if (field.omitTermFreqAndPositions()) {
if (field.omitPositions()) {
f.setIndexOptions(IndexOptions.DOCS_ONLY);
} else {
f.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
}
} else {
f.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
}
f.setBoost(boost);
return f;
}
