public static <T> void populateOutputTupleEntry( CombinerDefinition<T> definition, TupleEntry output, Tuple resultTuple) { // set the ID so we can differentiate later output.setRaw(MultiCombiner.ID_FIELD, definition.getId()); // our tuples are of the form groupFields+outputFields, set the TupleEntry fields appropriately Fields groupFields = definition.getGroupFields(); int index = 0; for (int i = 0; i < groupFields.size(); i++) { output.setRaw(groupFields.get(i), resultTuple.getObject(index)); index++; } Fields outputFields = definition.getOutputFields(); for (int i = 0; i < outputFields.size(); i++) { output.setRaw(outputFields.get(i), resultTuple.getObject(index)); index++; } }
static List<String> asStrings(Fields fields) { if (fields == null || !fields.isDefined()) { // use auto-generated name return Collections.emptyList(); } int size = fields.size(); List<String> names = new ArrayList<String>(size); for (int fieldIndex = 0; fieldIndex < size; fieldIndex++) { names.add(fields.get(fieldIndex).toString()); } return names; }
public static void validate(File solrCoreDir, String dataDirPropertyName, Fields schemeFields) throws IOException { // Verify solrHomeDir exists if (!solrCoreDir.exists() || !solrCoreDir.isDirectory()) { throw new TapException("Solr core directory doesn't exist: " + solrCoreDir); } File tmpSolrHome = makeTempSolrHome(solrCoreDir); // Set up a temp location for Solr home, where we're write out a synthetic solr.xml // that references the core directory. String coreName = solrCoreDir.getName(); String corePath = solrCoreDir.getAbsolutePath(); String solrXmlContent = String.format( "<solr><cores><core name=\"%s\" instanceDir=\"%s\"></core></cores></solr>", coreName, corePath); File solrXmlFile = new File(tmpSolrHome, "solr.xml"); FileUtils.write(solrXmlFile, solrXmlContent); // Set up a temp location for data, so when we instantiate the coreContainer, // we don't pollute the solr home with a /data sub-dir. String tmpFolder = System.getProperty("java.io.tmpdir"); File tmpDataDir = new File(tmpFolder, UUID.randomUUID().toString()); tmpDataDir.mkdir(); System.setProperty("solr.solr.home", tmpSolrHome.getAbsolutePath()); System.setProperty(dataDirPropertyName, tmpDataDir.getAbsolutePath()); System.setProperty( "enable.special-handlers", "false"); // All we need is the update request handler System.setProperty( "enable.cache-warming", "false"); // We certainly don't need to warm the cache CoreContainer.Initializer initializer = new CoreContainer.Initializer(); CoreContainer coreContainer = null; try { coreContainer = initializer.initialize(); Collection<SolrCore> cores = coreContainer.getCores(); SolrCore core = null; if (cores.size() == 0) { throw new TapException("No Solr cores are available"); } else if (cores.size() > 1) { throw new TapException("Only one Solr core is supported"); } else { core = cores.iterator().next(); } IndexSchema schema = core.getSchema(); Map<String, SchemaField> solrFields = schema.getFields(); Set<String> schemeFieldnames = new HashSet<String>(); for (int i = 0; i < schemeFields.size(); i++) { String fieldName = schemeFields.get(i).toString(); if (!solrFields.containsKey(fieldName)) { throw new TapException("Sink field name doesn't exist in Solr schema: " + fieldName); } schemeFieldnames.add(fieldName); } for (String solrFieldname : solrFields.keySet()) { SchemaField solrField = solrFields.get(solrFieldname); if (solrField.isRequired() && !schemeFieldnames.contains(solrFieldname)) { throw new TapException("No sink field name for required Solr field: " + solrFieldname); } } } finally { if (coreContainer != null) { coreContainer.shutdown(); } } }