Ejemplo n.º 1
0
  public void runParseTest(
      String fieldTerminator,
      String lineTerminator,
      String encloser,
      String escape,
      boolean encloseRequired)
      throws IOException {

    ClassLoader prevClassLoader = null;

    String[] argv =
        getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
    runImport(argv);
    try {
      String tableClassName = getTableName();

      argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
      SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);

      CompilationManager compileMgr = new CompilationManager(opts);
      String jarFileName = compileMgr.getJarFilename();

      // Make sure the user's class is loaded into our address space.
      prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);

      JobConf job = new JobConf();
      job.setJar(jarFileName);

      // Tell the job what class we're testing.
      job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);

      // use local mode in the same JVM.
      ConfigurationHelper.setJobtrackerAddr(job, "local");
      if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
        job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
      }
      String warehouseDir = getWarehouseDir();
      Path warehousePath = new Path(warehouseDir);
      Path inputPath = new Path(warehousePath, getTableName());
      Path outputPath = new Path(warehousePath, getTableName() + "-out");

      job.setMapperClass(ReparseMapper.class);
      job.setNumReduceTasks(0);
      FileInputFormat.addInputPath(job, inputPath);
      FileOutputFormat.setOutputPath(job, outputPath);

      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(NullWritable.class);

      JobClient.runJob(job);
    } catch (InvalidOptionsException ioe) {
      fail(ioe.toString());
    } catch (ParseException pe) {
      fail(pe.toString());
    } finally {
      if (null != prevClassLoader) {
        ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
      }
    }
  }
Ejemplo n.º 2
0
  private void doExternalTest(final byte[] data, final String filename) throws IOException {

    Configuration conf = new Configuration();
    if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);
    String tmpDir = System.getProperty("test.build.data", "/tmp/");

    Path tmpPath = new Path(tmpDir);
    Path blobFile = new Path(tmpPath, filename);

    // make any necessary parent dirs.
    Path blobParent = blobFile.getParent();
    if (!fs.exists(blobParent)) {
      fs.mkdirs(blobParent);
    }

    LobFile.Writer lw = LobFile.create(blobFile, conf, false);
    try {
      long off = lw.tell();
      long len = data.length;
      OutputStream os = lw.writeBlobRecord(len);
      os.write(data, 0, data.length);
      os.close();
      lw.close();

      String refString = "externalLob(lf," + filename + "," + off + "," + len + ")";
      BlobRef blob = BlobRef.parse(refString);
      assertTrue(blob.isExternal());
      assertEquals(refString, blob.toString());
      InputStream is = blob.getDataStream(conf, tmpPath);
      assertNotNull(is);

      byte[] buf = new byte[4096];
      int bytes = is.read(buf, 0, 4096);
      is.close();

      assertEquals(data.length, bytes);
      for (int i = 0; i < bytes; i++) {
        assertEquals(data[i], buf[i]);
      }
    } finally {
      fs.delete(blobFile, false);
    }
  }