@SuppressWarnings({"rawtypes", "unchecked"})
@Test(expectedExceptions = SqoopException.class)
public void testIncorrectSchemaColumnSize() throws Exception {
MutableContext context = new MutableMapContext();
LinkConfiguration linkConfig = new LinkConfiguration();
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
context.setString(
GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
+ "(SELECT * FROM "
+ executor.delimitIdentifier(tableName)
+ " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS");
GenericJdbcPartition partition = new GenericJdbcPartition();
Extractor extractor = new GenericJdbcExtractor();
DummyWriter writer = new DummyWriter();
Schema schema = new Schema("TestIncorrectColumns");
ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);
partition.setConditions("-50 <= ICOL AND ICOL < -16");
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
@Test
public void testNullValueExtracted() throws Exception {
if (!executor.existTable(nullDataTableName)) {
executor.executeUpdate(
"CREATE TABLE "
+ executor.delimitIdentifier(nullDataTableName)
+ "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20), DATECOL DATE)");
for (int i = 0; i < NUMBER_OF_ROWS; i++) {
int value = i;
String sql =
"INSERT INTO "
+ executor.delimitIdentifier(nullDataTableName)
+ " VALUES("
+ value
+ ",null,null,null)";
executor.executeUpdate(sql);
}
}
MutableContext context = new MutableMapContext();
LinkConfiguration linkConfig = new LinkConfiguration();
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
context.setString(
GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
"SELECT * FROM " + executor.delimitIdentifier(nullDataTableName) + " WHERE ${CONDITIONS}");
Extractor extractor = new GenericJdbcExtractor();
DummyNullDataWriter writer = new DummyNullDataWriter();
Schema schema = new Schema("TestExtractor");
schema
.addColumn(new FixedPoint("c1", 2L, true))
.addColumn(new Decimal("c2", 5, 2))
.addColumn(new Text("c3"))
.addColumn(new Date("c4"));
ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);
GenericJdbcPartition partition = new GenericJdbcPartition();
partition.setConditions("-50 <= ICOL AND ICOL < -16");
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Test
public void testSubquery() throws Exception {
MutableContext context = new MutableMapContext();
LinkConfiguration linkConfig = new LinkConfiguration();
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
context.setString(
GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL,SQOOP_SUBQUERY_ALIAS.DATECOL FROM "
+ "(SELECT * FROM "
+ executor.delimitIdentifier(tableName)
+ " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS");
GenericJdbcPartition partition;
Extractor extractor = new GenericJdbcExtractor();
DummyWriter writer = new DummyWriter();
Schema schema = new Schema("TestExtractor");
// dummy columns added, all we need is the column count to match to the
// result set
schema
.addColumn(new FixedPoint("c1", 2L, true))
.addColumn(new Text("c2"))
.addColumn(new Date("c3"));
ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);
partition = new GenericJdbcPartition();
partition.setConditions("-50 <= ICOL AND ICOL < -16");
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("-16 <= ICOL AND ICOL < 17");
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("17 <= ICOL AND ICOL < 50");
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public void run(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
String extractorName = conf.get(MRJobConstants.JOB_ETL_EXTRACTOR);
Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName);
Schema fromSchema = MRConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
Schema toSchema = MRConfigurationUtils.getConnectorSchema(Direction.TO, conf);
matcher = MatcherFactory.getMatcher(fromSchema, toSchema);
String fromIDFClass = conf.get(MRJobConstants.FROM_INTERMEDIATE_DATA_FORMAT);
fromIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(fromIDFClass);
fromIDF.setSchema(matcher.getFromSchema());
String toIDFClass = conf.get(MRJobConstants.TO_INTERMEDIATE_DATA_FORMAT);
toIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(toIDFClass);
toIDF.setSchema(matcher.getToSchema());
// Objects that should be passed to the Executor execution
PrefixContext subContext =
new PrefixContext(conf, MRJobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
Object fromConfig = MRConfigurationUtils.getConnectorLinkConfig(Direction.FROM, conf);
Object fromJob = MRConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf);
SqoopSplit split = context.getCurrentKey();
ExtractorContext extractorContext =
new ExtractorContext(
subContext, new SqoopDataWriter(context, fromIDF, toIDF, matcher), fromSchema);
try {
LOG.info("Running extractor class " + extractorName);
extractor.extract(extractorContext, fromConfig, fromJob, split.getPartition());
LOG.info("Extractor has finished");
context.getCounter(SqoopCounters.ROWS_READ).increment(extractor.getRowsRead());
} catch (Exception e) {
throw new SqoopException(MRExecutionError.MAPRED_EXEC_0017, e);
} finally {
LOG.info("Stopping progress service");
}
}
