public void runParseTest( String fieldTerminator, String lineTerminator, String encloser, String escape, boolean encloseRequired) throws IOException { ClassLoader prevClassLoader = null; String[] argv = getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired); runImport(argv); try { String tableClassName = getTableName(); argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired); SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true); CompilationManager compileMgr = new CompilationManager(opts); String jarFileName = compileMgr.getJarFilename(); // Make sure the user's class is loaded into our address space. prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName); JobConf job = new JobConf(); job.setJar(jarFileName); // Tell the job what class we're testing. job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName); // use local mode in the same JVM. ConfigurationHelper.setJobtrackerAddr(job, "local"); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } String warehouseDir = getWarehouseDir(); Path warehousePath = new Path(warehouseDir); Path inputPath = new Path(warehousePath, getTableName()); Path outputPath = new Path(warehousePath, getTableName() + "-out"); job.setMapperClass(ReparseMapper.class); job.setNumReduceTasks(0); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); JobClient.runJob(job); } catch (InvalidOptionsException ioe) { fail(ioe.toString()); } catch (ParseException pe) { fail(pe.toString()); } finally { if (null != prevClassLoader) { ClassLoaderStack.setCurrentClassLoader(prevClassLoader); } } }
/** Export some rows from a SequenceFile, make sure they import correctly. */ public void testSequenceFileExport() throws Exception { final int TOTAL_RECORDS = 10; // First, generate class and jar files that represent the table // we're exporting to. LOG.info("Creating initial schema for SeqFile test"); createTable(); LOG.info("Generating code..."); CodeGenTool codeGen = new CodeGenTool(); String[] codeGenArgs = getCodeGenArgv(); SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null, true); codeGen.validateOptions(options); int ret = codeGen.run(options); assertEquals(0, ret); List<String> generatedJars = codeGen.getGeneratedJarFiles(); // Now, wipe the created table so we can export on top of it again. LOG.info("Resetting schema and data..."); createTable(); // Wipe the directory we use when creating files to export to ensure // it's ready for new SequenceFiles. removeTablePath(); assertNotNull(generatedJars); assertEquals("Expected 1 generated jar file", 1, generatedJars.size()); String jarFileName = generatedJars.get(0); // Sqoop generates jars named "foo.jar"; by default, this should contain a // class named 'foo'. Extract the class name. Path jarPath = new Path(jarFileName); String jarBaseName = jarPath.getName(); assertTrue(jarBaseName.endsWith(".jar")); assertTrue(jarBaseName.length() > ".jar".length()); String className = jarBaseName.substring(0, jarBaseName.length() - ".jar".length()); LOG.info("Using jar filename: " + jarFileName); LOG.info("Using class name: " + className); ClassLoader prevClassLoader = null; try { if (null != jarFileName) { prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, className); } // Now use this class and jar name to create a sequence file. LOG.info("Writing data to SequenceFiles"); createSequenceFile(0, TOTAL_RECORDS, className); // Now run and verify the export. LOG.info("Exporting SequenceFile-based data"); runExport(getArgv(true, 10, 10, "--class-name", className, "--jar-file", jarFileName)); verifyExport(TOTAL_RECORDS); } finally { if (null != prevClassLoader) { ClassLoaderStack.setCurrentClassLoader(prevClassLoader); } } }
public void testMerge() throws Exception { createTable(); // Create a jar to use for the merging process; we'll load it // into the current thread CL for when this runs. This needs // to contain a different class name than used for the imports // due to classloaderstack issues in the same JVM. final String MERGE_CLASS_NAME = "ClassForMerging"; SqoopOptions options = getSqoopOptions(newConf()); options.setTableName(TABLE_NAME); options.setClassName(MERGE_CLASS_NAME); CodeGenTool codeGen = new CodeGenTool(); Sqoop codeGenerator = new Sqoop(codeGen, options.getConf(), options); int ret = Sqoop.runSqoop(codeGenerator, new String[0]); if (0 != ret) { fail("Nonzero exit from codegen: " + ret); } List<String> jars = codeGen.getGeneratedJarFiles(); String jarFileName = jars.get(0); // Now do the imports. Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); options = getSqoopOptions(newConf()); options.setTableName(TABLE_NAME); options.setNumMappers(1); // Do an import of this data into the "old" dataset. options.setTargetDir(new Path(warehouse, "merge-old").toString()); options.setIncrementalMode(IncrementalMode.DateLastModified); options.setIncrementalTestColumn("lastmod"); ImportTool importTool = new ImportTool(); Sqoop importer = new Sqoop(importTool, options.getConf(), options); ret = Sqoop.runSqoop(importer, new String[0]); if (0 != ret) { fail("Initial import failed with exit code " + ret); } // Check that we got records that meet our expected values. assertRecordStartsWith("0,0,", "merge-old"); assertRecordStartsWith("1,42,", "merge-old"); long prevImportEnd = System.currentTimeMillis(); Thread.sleep(25); // Modify the data in the warehouse. PreparedStatement s = conn.prepareStatement("UPDATE " + TABLE_NAME + " SET val=43, lastmod=NOW() WHERE id=1"); try { s.executeUpdate(); conn.commit(); } finally { s.close(); } s = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (" + "3,313,NOW())"); try { s.executeUpdate(); conn.commit(); } finally { s.close(); } Thread.sleep(25); // Do another import, into the "new" dir. options = getSqoopOptions(newConf()); options.setTableName(TABLE_NAME); options.setNumMappers(1); options.setTargetDir(new Path(warehouse, "merge-new").toString()); options.setIncrementalMode(IncrementalMode.DateLastModified); options.setIncrementalTestColumn("lastmod"); options.setIncrementalLastValue(new Timestamp(prevImportEnd).toString()); importTool = new ImportTool(); importer = new Sqoop(importTool, options.getConf(), options); ret = Sqoop.runSqoop(importer, new String[0]); if (0 != ret) { fail("Second import failed with exit code " + ret); } assertRecordStartsWith("1,43,", "merge-new"); assertRecordStartsWith("3,313,", "merge-new"); // Now merge the results! ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME); options = getSqoopOptions(newConf()); options.setMergeOldPath(new Path(warehouse, "merge-old").toString()); options.setMergeNewPath(new Path(warehouse, "merge-new").toString()); options.setMergeKeyCol("ID"); options.setTargetDir(new Path(warehouse, "merge-final").toString()); options.setClassName(MERGE_CLASS_NAME); MergeTool mergeTool = new MergeTool(); Sqoop merger = new Sqoop(mergeTool, options.getConf(), options); ret = Sqoop.runSqoop(merger, new String[0]); if (0 != ret) { fail("Merge failed with exit code " + ret); } assertRecordStartsWith("0,0,", "merge-final"); assertRecordStartsWith("1,43,", "merge-final"); assertRecordStartsWith("3,313,", "merge-final"); }