public void runParseTest( String fieldTerminator, String lineTerminator, String encloser, String escape, boolean encloseRequired) throws IOException { ClassLoader prevClassLoader = null; String[] argv = getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired); runImport(argv); try { String tableClassName = getTableName(); argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired); SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true); CompilationManager compileMgr = new CompilationManager(opts); String jarFileName = compileMgr.getJarFilename(); // Make sure the user's class is loaded into our address space. prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName); JobConf job = new JobConf(); job.setJar(jarFileName); // Tell the job what class we're testing. job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName); // use local mode in the same JVM. ConfigurationHelper.setJobtrackerAddr(job, "local"); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } String warehouseDir = getWarehouseDir(); Path warehousePath = new Path(warehouseDir); Path inputPath = new Path(warehousePath, getTableName()); Path outputPath = new Path(warehousePath, getTableName() + "-out"); job.setMapperClass(ReparseMapper.class); job.setNumReduceTasks(0); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); JobClient.runJob(job); } catch (InvalidOptionsException ioe) { fail(ioe.toString()); } catch (ParseException pe) { fail(pe.toString()); } finally { if (null != prevClassLoader) { ClassLoaderStack.setCurrentClassLoader(prevClassLoader); } } }
private void doExternalTest(final byte[] data, final String filename) throws IOException { Configuration conf = new Configuration(); if (!BaseSqoopTestCase.isOnPhysicalCluster()) { conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } FileSystem fs = FileSystem.get(conf); String tmpDir = System.getProperty("test.build.data", "/tmp/"); Path tmpPath = new Path(tmpDir); Path blobFile = new Path(tmpPath, filename); // make any necessary parent dirs. Path blobParent = blobFile.getParent(); if (!fs.exists(blobParent)) { fs.mkdirs(blobParent); } LobFile.Writer lw = LobFile.create(blobFile, conf, false); try { long off = lw.tell(); long len = data.length; OutputStream os = lw.writeBlobRecord(len); os.write(data, 0, data.length); os.close(); lw.close(); String refString = "externalLob(lf," + filename + "," + off + "," + len + ")"; BlobRef blob = BlobRef.parse(refString); assertTrue(blob.isExternal()); assertEquals(refString, blob.toString()); InputStream is = blob.getDataStream(conf, tmpPath); assertNotNull(is); byte[] buf = new byte[4096]; int bytes = is.read(buf, 0, 4096); is.close(); assertEquals(data.length, bytes); for (int i = 0; i < bytes; i++) { assertEquals(data[i], buf[i]); } } finally { fs.delete(blobFile, false); } }