public DocBuilder( DataImporter dataImporter, SolrWriter writer, DataImporter.RequestParams reqParams) { INSTANCE.set(this); this.dataImporter = dataImporter; this.writer = writer; DataImporter.QUERY_COUNT.set(importStatistics.queryCount); requestParameters = reqParams; verboseDebug = requestParameters.debug && requestParameters.verbose; functionsNamespace = EvaluatorBag.getFunctionsNamespace(this.dataImporter.getConfig().functions, this); persistedProperties = writer.readIndexerProperties(); }
private void runAThread(ThreadedEntityProcessorWrapper epw, EntityRow rows, String currProcess) throws Exception { currentEntityProcWrapper.set(epw); epw.threadedInit(context); initEntity(); try { epw.init(rows); DocWrapper docWrapper = this.docWrapper; Context.CURRENT_CONTEXT.set(context); for (; ; ) { if (DocBuilder.this.stop.get()) break; try { Map<String, Object> arow = epw.nextRow(); if (arow == null) { break; } else { importStatistics.rowsCount.incrementAndGet(); if (docWrapper == null && entity.isDocRoot) { docWrapper = new DocWrapper(); context.setDoc(docWrapper); DataConfig.Entity e = entity.parentEntity; for (EntityRow row = rows; row != null && e != null; row = row.tail, e = e.parentEntity) { addFields(e, docWrapper, row.row, epw.resolver); } } if (docWrapper != null) { handleSpecialCommands(arow, docWrapper); addFields(entity, docWrapper, arow, epw.resolver); } if (entity.entities != null) { EntityRow nextRow = new EntityRow(arow, rows, entity.name); for (DataConfig.Entity e : entity.entities) { epw.children.get(e).run(docWrapper, currProcess, nextRow); } } } if (entity.isDocRoot) { LOG.info("a row on docroot" + docWrapper); if (!docWrapper.isEmpty()) { LOG.info("adding a doc " + docWrapper); boolean result = writer.upload(docWrapper); docWrapper = null; if (result) { importStatistics.docCount.incrementAndGet(); } else { importStatistics.failedDocCount.incrementAndGet(); } } } } catch (DataImportHandlerException dihe) { exception = dihe; if (dihe.getErrCode() == SKIP_ROW || dihe.getErrCode() == SKIP) { importStatistics.skipDocCount.getAndIncrement(); exception = null; // should not propogate up continue; } if (entity.isDocRoot) { if (dihe.getErrCode() == DataImportHandlerException.SKIP) { importStatistics.skipDocCount.getAndIncrement(); exception = null; // should not propogate up } else { LOG.error( "Exception while processing: " + entity.name + " document : " + docWrapper, dihe); } if (dihe.getErrCode() == DataImportHandlerException.SEVERE) throw dihe; } else { // if this is not the docRoot then the execution has happened in the same thread. so // propogate up, // it will be handled at the docroot entityEnded.set(true); throw dihe; } entityEnded.set(true); } } } finally { epw.destroy(); currentEntityProcWrapper.remove(); Context.CURRENT_CONTEXT.remove(); } }