@Override
public Reader getData(String query) {
URL url = null;
try {
if (URIMETHOD.matcher(query).find()) url = new URL(query);
else url = new URL(baseUrl + query);
LOG.debug("Accessing URL: " + url.toString());
URLConnection conn = url.openConnection();
conn.setConnectTimeout(connectionTimeout);
conn.setReadTimeout(readTimeout);
InputStream in = conn.getInputStream();
String enc = encoding;
if (enc == null) {
String cType = conn.getContentType();
if (cType != null) {
Matcher m = CHARSET_PATTERN.matcher(cType);
if (m.find()) {
enc = m.group(1);
}
}
}
if (enc == null) enc = UTF_8;
DataImporter.QUERY_COUNT.get().incrementAndGet();
return new InputStreamReader(in, enc);
} catch (Exception e) {
LOG.error("Exception thrown while getting data", e);
throw new DataImportHandlerException(
DataImportHandlerException.SEVERE, "Exception in invoking url " + url, e);
}
}
public DocBuilder(
DataImporter dataImporter, SolrWriter writer, DataImporter.RequestParams reqParams) {
INSTANCE.set(this);
this.dataImporter = dataImporter;
this.writer = writer;
DataImporter.QUERY_COUNT.set(importStatistics.queryCount);
requestParameters = reqParams;
verboseDebug = requestParameters.debug && requestParameters.verbose;
functionsNamespace =
EvaluatorBag.getFunctionsNamespace(this.dataImporter.getConfig().functions, this);
persistedProperties = writer.readIndexerProperties();
}
