Пример #1
0
  public void runParseTest(
      String fieldTerminator,
      String lineTerminator,
      String encloser,
      String escape,
      boolean encloseRequired)
      throws IOException {

    ClassLoader prevClassLoader = null;

    String[] argv =
        getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
    runImport(argv);
    try {
      String tableClassName = getTableName();

      argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
      SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);

      CompilationManager compileMgr = new CompilationManager(opts);
      String jarFileName = compileMgr.getJarFilename();

      // Make sure the user's class is loaded into our address space.
      prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);

      JobConf job = new JobConf();
      job.setJar(jarFileName);

      // Tell the job what class we're testing.
      job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);

      // use local mode in the same JVM.
      ConfigurationHelper.setJobtrackerAddr(job, "local");
      if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
        job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
      }
      String warehouseDir = getWarehouseDir();
      Path warehousePath = new Path(warehouseDir);
      Path inputPath = new Path(warehousePath, getTableName());
      Path outputPath = new Path(warehousePath, getTableName() + "-out");

      job.setMapperClass(ReparseMapper.class);
      job.setNumReduceTasks(0);
      FileInputFormat.addInputPath(job, inputPath);
      FileOutputFormat.setOutputPath(job, outputPath);

      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(NullWritable.class);

      JobClient.runJob(job);
    } catch (InvalidOptionsException ioe) {
      fail(ioe.toString());
    } catch (ParseException pe) {
      fail(pe.toString());
    } finally {
      if (null != prevClassLoader) {
        ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
      }
    }
  }
Пример #2
0
  /** Export some rows from a SequenceFile, make sure they import correctly. */
  public void testSequenceFileExport() throws Exception {

    final int TOTAL_RECORDS = 10;

    // First, generate class and jar files that represent the table
    // we're exporting to.
    LOG.info("Creating initial schema for SeqFile test");
    createTable();
    LOG.info("Generating code...");
    CodeGenTool codeGen = new CodeGenTool();
    String[] codeGenArgs = getCodeGenArgv();
    SqoopOptions options = codeGen.parseArguments(codeGenArgs, null, null, true);
    codeGen.validateOptions(options);
    int ret = codeGen.run(options);
    assertEquals(0, ret);
    List<String> generatedJars = codeGen.getGeneratedJarFiles();

    // Now, wipe the created table so we can export on top of it again.
    LOG.info("Resetting schema and data...");
    createTable();

    // Wipe the directory we use when creating files to export to ensure
    // it's ready for new SequenceFiles.
    removeTablePath();

    assertNotNull(generatedJars);
    assertEquals("Expected 1 generated jar file", 1, generatedJars.size());
    String jarFileName = generatedJars.get(0);
    // Sqoop generates jars named "foo.jar"; by default, this should contain a
    // class named 'foo'. Extract the class name.
    Path jarPath = new Path(jarFileName);
    String jarBaseName = jarPath.getName();
    assertTrue(jarBaseName.endsWith(".jar"));
    assertTrue(jarBaseName.length() > ".jar".length());
    String className = jarBaseName.substring(0, jarBaseName.length() - ".jar".length());

    LOG.info("Using jar filename: " + jarFileName);
    LOG.info("Using class name: " + className);

    ClassLoader prevClassLoader = null;

    try {
      if (null != jarFileName) {
        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, className);
      }

      // Now use this class and jar name to create a sequence file.
      LOG.info("Writing data to SequenceFiles");
      createSequenceFile(0, TOTAL_RECORDS, className);

      // Now run and verify the export.
      LOG.info("Exporting SequenceFile-based data");
      runExport(getArgv(true, 10, 10, "--class-name", className, "--jar-file", jarFileName));
      verifyExport(TOTAL_RECORDS);
    } finally {
      if (null != prevClassLoader) {
        ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
      }
    }
  }
Пример #3
0
  public void testMerge() throws Exception {
    createTable();

    // Create a jar to use for the merging process; we'll load it
    // into the current thread CL for when this runs. This needs
    // to contain a different class name than used for the imports
    // due to classloaderstack issues in the same JVM.
    final String MERGE_CLASS_NAME = "ClassForMerging";
    SqoopOptions options = getSqoopOptions(newConf());
    options.setTableName(TABLE_NAME);
    options.setClassName(MERGE_CLASS_NAME);

    CodeGenTool codeGen = new CodeGenTool();
    Sqoop codeGenerator = new Sqoop(codeGen, options.getConf(), options);
    int ret = Sqoop.runSqoop(codeGenerator, new String[0]);
    if (0 != ret) {
      fail("Nonzero exit from codegen: " + ret);
    }

    List<String> jars = codeGen.getGeneratedJarFiles();
    String jarFileName = jars.get(0);

    // Now do the imports.

    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);

    options = getSqoopOptions(newConf());
    options.setTableName(TABLE_NAME);
    options.setNumMappers(1);

    // Do an import of this data into the "old" dataset.
    options.setTargetDir(new Path(warehouse, "merge-old").toString());
    options.setIncrementalMode(IncrementalMode.DateLastModified);
    options.setIncrementalTestColumn("lastmod");

    ImportTool importTool = new ImportTool();
    Sqoop importer = new Sqoop(importTool, options.getConf(), options);
    ret = Sqoop.runSqoop(importer, new String[0]);
    if (0 != ret) {
      fail("Initial import failed with exit code " + ret);
    }

    // Check that we got records that meet our expected values.
    assertRecordStartsWith("0,0,", "merge-old");
    assertRecordStartsWith("1,42,", "merge-old");

    long prevImportEnd = System.currentTimeMillis();

    Thread.sleep(25);

    // Modify the data in the warehouse.
    PreparedStatement s =
        conn.prepareStatement("UPDATE " + TABLE_NAME + " SET val=43, lastmod=NOW() WHERE id=1");
    try {
      s.executeUpdate();
      conn.commit();
    } finally {
      s.close();
    }

    s = conn.prepareStatement("INSERT INTO " + TABLE_NAME + " VALUES (" + "3,313,NOW())");
    try {
      s.executeUpdate();
      conn.commit();
    } finally {
      s.close();
    }

    Thread.sleep(25);

    // Do another import, into the "new" dir.
    options = getSqoopOptions(newConf());
    options.setTableName(TABLE_NAME);
    options.setNumMappers(1);
    options.setTargetDir(new Path(warehouse, "merge-new").toString());
    options.setIncrementalMode(IncrementalMode.DateLastModified);
    options.setIncrementalTestColumn("lastmod");
    options.setIncrementalLastValue(new Timestamp(prevImportEnd).toString());

    importTool = new ImportTool();
    importer = new Sqoop(importTool, options.getConf(), options);
    ret = Sqoop.runSqoop(importer, new String[0]);
    if (0 != ret) {
      fail("Second import failed with exit code " + ret);
    }

    assertRecordStartsWith("1,43,", "merge-new");
    assertRecordStartsWith("3,313,", "merge-new");

    // Now merge the results!
    ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME);

    options = getSqoopOptions(newConf());
    options.setMergeOldPath(new Path(warehouse, "merge-old").toString());
    options.setMergeNewPath(new Path(warehouse, "merge-new").toString());
    options.setMergeKeyCol("ID");
    options.setTargetDir(new Path(warehouse, "merge-final").toString());
    options.setClassName(MERGE_CLASS_NAME);

    MergeTool mergeTool = new MergeTool();
    Sqoop merger = new Sqoop(mergeTool, options.getConf(), options);
    ret = Sqoop.runSqoop(merger, new String[0]);
    if (0 != ret) {
      fail("Merge failed with exit code " + ret);
    }

    assertRecordStartsWith("0,0,", "merge-final");
    assertRecordStartsWith("1,43,", "merge-final");
    assertRecordStartsWith("3,313,", "merge-final");
  }