public void runParseTest(
String fieldTerminator,
String lineTerminator,
String encloser,
String escape,
boolean encloseRequired)
throws IOException {
ClassLoader prevClassLoader = null;
String[] argv =
getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
runImport(argv);
try {
String tableClassName = getTableName();
argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);
CompilationManager compileMgr = new CompilationManager(opts);
String jarFileName = compileMgr.getJarFilename();
// Make sure the user's class is loaded into our address space.
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);
JobConf job = new JobConf();
job.setJar(jarFileName);
// Tell the job what class we're testing.
job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
// use local mode in the same JVM.
ConfigurationHelper.setJobtrackerAddr(job, "local");
if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
}
String warehouseDir = getWarehouseDir();
Path warehousePath = new Path(warehouseDir);
Path inputPath = new Path(warehousePath, getTableName());
Path outputPath = new Path(warehousePath, getTableName() + "-out");
job.setMapperClass(ReparseMapper.class);
job.setNumReduceTasks(0);
FileInputFormat.addInputPath(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
JobClient.runJob(job);
} catch (InvalidOptionsException ioe) {
fail(ioe.toString());
} catch (ParseException pe) {
fail(pe.toString());
} finally {
if (null != prevClassLoader) {
ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
}
}
}
/** Export some rows from a SequenceFile, make sure they import correctly. */
public void testSequenceFileExport() throws Exception {
final int TOTAL_RECORDS = 10;
// First, generate class and jar files that represent the table
// we're exporting to.
LOG.info("Creating initial schema for SeqFile test");
createTable();
LOG.info("Generating code...");
CodeGenTool codeGen = new CodeGenTool();
String[] codeGenArgs = getCodeGenArgv();
SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null, true);
codeGen.validateOptions(options);
int ret = codeGen.run(options);
assertEquals(0, ret);
List<String> generatedJars = codeGen.getGeneratedJarFiles();
// Now, wipe the created table so we can export on top of it again.
LOG.info("Resetting schema and data...");
createTable();
// Wipe the directory we use when creating files to export to ensure
// it's ready for new SequenceFiles.
removeTablePath();
assertNotNull(generatedJars);
assertEquals("Expected 1 generated jar file", 1, generatedJars.size());
String jarFileName = generatedJars.get(0);
// Sqoop generates jars named "foo.jar"; by default, this should contain a
// class named 'foo'. Extract the class name.
Path jarPath = new Path(jarFileName);
String jarBaseName = jarPath.getName();
assertTrue(jarBaseName.endsWith(".jar"));
assertTrue(jarBaseName.length() > ".jar".length());
String className = jarBaseName.substring(0, jarBaseName.length() - ".jar".length());
LOG.info("Using jar filename: " + jarFileName);
LOG.info("Using class name: " + className);
ClassLoader prevClassLoader = null;
try {
if (null != jarFileName) {
prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, className);
}
// Now use this class and jar name to create a sequence file.
LOG.info("Writing data to SequenceFiles");
createSequenceFile(0, TOTAL_RECORDS, className);
// Now run and verify the export.
LOG.info("Exporting SequenceFile-based data");
runExport(getArgv(true, 10, 10, "--class-name", className, "--jar-file", jarFileName));
verifyExport(TOTAL_RECORDS);
} finally {
if (null != prevClassLoader) {
ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
}
}
}
public void testMerge() throws Exception {
createTable();
// Create a jar to use for the merging process; we'll load it
// into the current thread CL for when this runs. This needs
// to contain a different class name than used for the imports
// due to classloaderstack issues in the same JVM.
final String MERGE_CLASS_NAME = "ClassForMerging";
SqoopOptions options = getSqoopOptions(newConf());
options.setTableName(TABLE_NAME);
options.setClassName(MERGE_CLASS_NAME);
CodeGenTool codeGen = new CodeGenTool();
Sqoop codeGenerator = new Sqoop(codeGen, options.getConf(), options);
int ret = Sqoop.runSqoop(codeGenerator, new String[0]);
if (0 != ret) {
fail("Nonzero exit from codegen: " + ret);
}
List<String> jars = codeGen.getGeneratedJarFiles();
String jarFileName = jars.get(0);
// Now do the imports.
Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
options = getSqoopOptions(newConf());
options.setTableName(TABLE_NAME);
options.setNumMappers(1);
// Do an import of this data into the "old" dataset.
options.setTargetDir(new Path(warehouse, "merge-old").toString());
options.setIncrementalMode(IncrementalMode.DateLastModified);
options.setIncrementalTestColumn("lastmod");
ImportTool importTool = new ImportTool();
Sqoop importer = new Sqoop(importTool, options.getConf(), options);
ret = Sqoop.runSqoop(importer, new String[0]);
if (0 != ret) {
fail("Initial import failed with exit code " + ret);
}
// Check that we got records that meet our expected values.
assertRecordStartsWith("0,0,", "merge-old");
assertRecordStartsWith("1,42,", "merge-old");
long prevImportEnd = System.currentTimeMillis();
Thread.sleep(25);
// Modify the data in the warehouse.
PreparedStatement s =
conn.prepareStatement("UPDATE " + TABLE_NAME + " SET val=43, lastmod=NOW() WHERE id=1");
try {
s.executeUpdate();
conn.commit();
} finally {
s.close();
}
s = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (" + "3,313,NOW())");
try {
s.executeUpdate();
conn.commit();
} finally {
s.close();
}
Thread.sleep(25);
// Do another import, into the "new" dir.
options = getSqoopOptions(newConf());
options.setTableName(TABLE_NAME);
options.setNumMappers(1);
options.setTargetDir(new Path(warehouse, "merge-new").toString());
options.setIncrementalMode(IncrementalMode.DateLastModified);
options.setIncrementalTestColumn("lastmod");
options.setIncrementalLastValue(new Timestamp(prevImportEnd).toString());
importTool = new ImportTool();
importer = new Sqoop(importTool, options.getConf(), options);
ret = Sqoop.runSqoop(importer, new String[0]);
if (0 != ret) {
fail("Second import failed with exit code " + ret);
}
assertRecordStartsWith("1,43,", "merge-new");
assertRecordStartsWith("3,313,", "merge-new");
// Now merge the results!
ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME);
options = getSqoopOptions(newConf());
options.setMergeOldPath(new Path(warehouse, "merge-old").toString());
options.setMergeNewPath(new Path(warehouse, "merge-new").toString());
options.setMergeKeyCol("ID");
options.setTargetDir(new Path(warehouse, "merge-final").toString());
options.setClassName(MERGE_CLASS_NAME);
MergeTool mergeTool = new MergeTool();
Sqoop merger = new Sqoop(mergeTool, options.getConf(), options);
ret = Sqoop.runSqoop(merger, new String[0]);
if (0 != ret) {
fail("Merge failed with exit code " + ret);
}
assertRecordStartsWith("0,0,", "merge-final");
assertRecordStartsWith("1,43,", "merge-final");
assertRecordStartsWith("3,313,", "merge-final");
}