/** this must be MT safe... may be called concurrently from multiple threads. */
void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template)
throws IOException {
// the line number is passed for error reporting in MT mode as well as for optional rowId.
// first, create the lucene document
for (int i = 0; i < vals.length; i++) {
if (adders[i] == null) continue; // skip this field
String val = vals[i];
adders[i].add(doc, line, i, val);
}
// add any literals
for (String fname : literals.keySet()) {
String val = literals.get(fname);
doc.addField(fname, val);
}
if (rowId != null) {
doc.addField(rowId, line + rowIdOffset);
}
template.solrDoc = doc;
processor.processAdd(template);
}
CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
this.processor = processor;
this.params = req.getParams();
this.literals = new HashMap<>();
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite = params.getBool(OVERWRITE, true);
templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
strategy =
new CSVStrategy(
',',
'"',
CSVStrategy.COMMENTS_DISABLED,
CSVStrategy.ESCAPE_DISABLED,
false,
false,
false,
true);
String sep = params.get(SEPARATOR);
if (sep != null) {
if (sep.length() != 1)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Invalid separator:'" + sep + "'");
strategy.setDelimiter(sep.charAt(0));
}
String encapsulator = params.get(ENCAPSULATOR);
if (encapsulator != null) {
if (encapsulator.length() != 1)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Invalid encapsulator:'" + encapsulator + "'");
}
String escape = params.get(ESCAPE);
if (escape != null) {
if (escape.length() != 1)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Invalid escape:'" + escape + "'");
}
rowId = params.get(ROW_ID);
rowIdOffset = params.getInt(ROW_ID_OFFSET, 0);
// if only encapsulator or escape is set, disable the other escaping mechanism
if (encapsulator == null && escape != null) {
strategy.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED);
strategy.setEscape(escape.charAt(0));
} else {
if (encapsulator != null) {
strategy.setEncapsulator(encapsulator.charAt(0));
}
if (escape != null) {
char ch = escape.charAt(0);
strategy.setEscape(ch);
if (ch == '\\') {
// If the escape is the standard backslash, then also enable
// unicode escapes (it's harmless since 'u' would not otherwise
// be escaped.
strategy.setUnicodeEscapeInterpretation(true);
}
}
}
String fn = params.get(FIELDNAMES);
fieldnames = fn != null ? commaSplit.split(fn, -1) : null;
Boolean hasHeader = params.getBool(HEADER);
skipLines = params.getInt(SKIPLINES, 0);
if (fieldnames == null) {
if (null == hasHeader) {
// assume the file has the headers if they aren't supplied in the args
hasHeader = true;
} else if (!hasHeader) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"CSVLoader: must specify fieldnames=<fields>* or header=true");
}
} else {
// if the fieldnames were supplied and the file has a header, we need to
// skip over that header.
if (hasHeader != null && hasHeader) skipLines++;
prepareFields();
}
}