public SqoopOutputFormatLoadExecutorSpark(JobContext jobctx) { context = jobctx; loaderName = context.getConfiguration().get(MRJobConstants.JOB_ETL_LOADER); writer = new SqoopRecordWriter(); // jackh: This must be conditional - Extract schema using credentials in case of MR and simply // extract from the // credentials object in case of Spark (due to known issue with Hadoop/Spark that the // credentials are never added // for serialization) // matcher = MatcherFactory.getMatcher( // MRConfigurationUtils.getConnectorSchema(Direction.FROM, context.getConfiguration()), // MRConfigurationUtils.getConnectorSchema(Direction.TO, context.getConfiguration())); matcher = MatcherFactory.getMatcher( MRConfigurationUtils.getConnectorSchemaUnsafe( Direction.FROM, context.getConfiguration()), MRConfigurationUtils.getConnectorSchemaUnsafe( Direction.TO, context.getConfiguration())); toDataFormat = (IntermediateDataFormat<?>) ClassUtils.instantiate( context.getConfiguration().get(MRJobConstants.TO_INTERMEDIATE_DATA_FORMAT)); // Using the TO schema since the SqoopDataWriter in the SqoopMapper encapsulates the // toDataFormat toDataFormat.setSchema(matcher.getToSchema()); }
@SuppressWarnings({"unchecked", "rawtypes"}) @Override public void run(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String extractorName = conf.get(MRJobConstants.JOB_ETL_EXTRACTOR); Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName); Schema fromSchema = MRConfigurationUtils.getConnectorSchema(Direction.FROM, conf); Schema toSchema = MRConfigurationUtils.getConnectorSchema(Direction.TO, conf); matcher = MatcherFactory.getMatcher(fromSchema, toSchema); String fromIDFClass = conf.get(MRJobConstants.FROM_INTERMEDIATE_DATA_FORMAT); fromIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(fromIDFClass); fromIDF.setSchema(matcher.getFromSchema()); String toIDFClass = conf.get(MRJobConstants.TO_INTERMEDIATE_DATA_FORMAT); toIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(toIDFClass); toIDF.setSchema(matcher.getToSchema()); // Objects that should be passed to the Executor execution PrefixContext subContext = new PrefixContext(conf, MRJobConstants.PREFIX_CONNECTOR_FROM_CONTEXT); Object fromConfig = MRConfigurationUtils.getConnectorLinkConfig(Direction.FROM, conf); Object fromJob = MRConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf); SqoopSplit split = context.getCurrentKey(); ExtractorContext extractorContext = new ExtractorContext( subContext, new SqoopDataWriter(context, fromIDF, toIDF, matcher), fromSchema); try { LOG.info("Running extractor class " + extractorName); extractor.extract(extractorContext, fromConfig, fromJob, split.getPartition()); LOG.info("Extractor has finished"); context.getCounter(SqoopCounters.ROWS_READ).increment(extractor.getRowsRead()); } catch (Exception e) { throw new SqoopException(MRExecutionError.MAPRED_EXEC_0017, e); } finally { LOG.info("Stopping progress service"); } }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public void run() { LOG.info("SqoopOutputFormatLoadExecutor consumer thread is starting"); try { DataReader reader = new SqoopOutputFormatDataReader(); Configuration conf = context.getConfiguration(); Loader loader = (Loader) ClassUtils.instantiate(loaderName); // Objects that should be passed to the Loader PrefixContext subContext = new PrefixContext(conf, MRJobConstants.PREFIX_CONNECTOR_TO_CONTEXT); Object connectorLinkConfig = MRConfigurationUtils.getConnectorLinkConfigUnsafe(Direction.TO, conf); Object connectorToJobConfig = MRConfigurationUtils.getConnectorJobConfigUnsafe(Direction.TO, conf); // Using the TO schema since the SqoopDataWriter in the SqoopMapper // encapsulates the toDataFormat // Create loader context LoaderContext loaderContext = new LoaderContext(subContext, reader, matcher.getToSchema()); LOG.info("Running loader class " + loaderName); loader.load(loaderContext, connectorLinkConfig, connectorToJobConfig); LOG.info("Loader has finished"); ((TaskAttemptContext) jobctx) .getCounter(SqoopCounters.ROWS_WRITTEN) .increment(loader.getRowsWritten()); } catch (Throwable t) { readerFinished = true; LOG.error("Error while loading data out of MR job.", t); // Release so that the writer can tell Sqoop something went // wrong. free.release(); throw new SqoopException(SparkExecutionError.SPARK_EXEC_0018, t); } // if no exception happens yet and reader finished before writer, // something went wrong if (!writerFinished) { // throw exception if data are not all consumed readerFinished = true; LOG.error("Reader terminated, but writer is still running!"); // Release so that the writer can tell Sqoop something went // wrong. free.release(); throw new SqoopException(SparkExecutionError.SPARK_EXEC_0019); } // inform writer that reader is finished readerFinished = true; }
@Override public void readFields(DataInput in) throws IOException { // read Partition class name String className = in.readUTF(); // instantiate Partition object Class<?> clz = ClassUtils.loadClass(className); if (clz == null) { throw new SqoopException(MRExecutionError.MAPRED_EXEC_0009, className); } try { partition = (Partition) clz.newInstance(); } catch (Exception e) { throw new SqoopException(MRExecutionError.MAPRED_EXEC_0010, className, e); } // read Partition object content partition.readFields(in); }
@Test public void testAddJarForClass() { jobRequest.addJarForClass(String.class); assertEquals(1, jobRequest.getJars().size()); assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(String.class))); }