public void runParseTest(
String fieldTerminator,
String lineTerminator,
String encloser,
String escape,
boolean encloseRequired)
throws IOException {
ClassLoader prevClassLoader = null;
String[] argv =
getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
runImport(argv);
try {
String tableClassName = getTableName();
argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);
CompilationManager compileMgr = new CompilationManager(opts);
String jarFileName = compileMgr.getJarFilename();
// Make sure the user's class is loaded into our address space.
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);
JobConf job = new JobConf();
job.setJar(jarFileName);
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
// use local mode in the same JVM.
ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
String warehouseDir = getWarehouseDir();
Path warehousePath = new Path(warehouseDir);
Path inputPath = new Path(warehousePath, getTableName());
Path outputPath = new Path(warehousePath, getTableName() + "-out");
job.setMapperClass(ReparseMapper.class);
job.setNumReduceTasks(0);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
JobClient.runJob(job);
} catch (InvalidOptionsException ioe) {
fail(ioe.toString());
} catch (ParseException pe) {
fail(pe.toString());
} finally {
if (null != prevClassLoader) {
ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
}
}
}
public List<InputSplit> split(Configuration conf, ResultSet results, String colName)
throws SQLException {
LOG.warn("Generating splits for a floating-point index column. Due to the");
LOG.warn("imprecise representation of floating-point values in Java, this");
LOG.warn("may result in an incomplete import.");
LOG.warn("You are strongly encouraged to choose an integral split column.");
List<InputSplit> splits = new ArrayList<InputSplit>();
if (results.getString(1) == null && results.getString(2) == null) {
// Range is null to null. Return a null split accordingly.
splits.add(
new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
colName + " IS NULL", colName + " IS NULL"));
return splits;
}
double minVal = results.getDouble(1);
double maxVal = results.getDouble(2);
// Use this as a hint. May need an extra task if the size doesn't
// divide cleanly.
int numSplits = ConfigurationHelper.getConfNumMaps(conf);
double splitSize = (maxVal - minVal) / (double) numSplits;
if (splitSize < MIN_INCREMENT) {
splitSize = MIN_INCREMENT;
}
String lowClausePrefix = colName + " >= ";
String highClausePrefix = colName + " < ";
double curLower = minVal;
double curUpper = curLower + splitSize;
while (curUpper < maxVal) {
splits.add(
new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
lowClausePrefix + Double.toString(curLower),
highClausePrefix + Double.toString(curUpper)));
curLower = curUpper;
curUpper += splitSize;
}
// Catch any overage and create the closed interval for the last split.
if (curLower <= maxVal || splits.size() == 1) {
splits.add(
new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
lowClausePrefix + Double.toString(curUpper),
colName + " <= " + Double.toString(maxVal)));
}
if (results.getString(1) == null || results.getString(2) == null) {
// At least one extrema is null; add a null split.
splits.add(
new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
colName + " IS NULL", colName + " IS NULL"));
}
return splits;
}