@SuppressWarnings({"rawtypes", "unchecked"})
  @Test(expectedExceptions = SqoopException.class)
  public void testIncorrectSchemaColumnSize() throws Exception {
    MutableContext context = new MutableMapContext();

    LinkConfiguration linkConfig = new LinkConfiguration();

    linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
    linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;

    FromJobConfiguration jobConfig = new FromJobConfiguration();

    context.setString(
        GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
        "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
            + "(SELECT * FROM "
            + executor.delimitIdentifier(tableName)
            + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS");

    GenericJdbcPartition partition = new GenericJdbcPartition();

    Extractor extractor = new GenericJdbcExtractor();
    DummyWriter writer = new DummyWriter();
    Schema schema = new Schema("TestIncorrectColumns");
    ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);

    partition.setConditions("-50 <= ICOL AND ICOL < -16");
    extractor.extract(extractorContext, linkConfig, jobConfig, partition);
  }
  @Test
  public void testNullValueExtracted() throws Exception {

    if (!executor.existTable(nullDataTableName)) {
      executor.executeUpdate(
          "CREATE TABLE "
              + executor.delimitIdentifier(nullDataTableName)
              + "(ICOL INTEGER PRIMARY KEY, DCOL DOUBLE, VCOL VARCHAR(20), DATECOL DATE)");

      for (int i = 0; i < NUMBER_OF_ROWS; i++) {
        int value = i;
        String sql =
            "INSERT INTO "
                + executor.delimitIdentifier(nullDataTableName)
                + " VALUES("
                + value
                + ",null,null,null)";
        executor.executeUpdate(sql);
      }
    }
    MutableContext context = new MutableMapContext();

    LinkConfiguration linkConfig = new LinkConfiguration();

    linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
    linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;

    FromJobConfiguration jobConfig = new FromJobConfiguration();
    context.setString(
        GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
        "SELECT * FROM " + executor.delimitIdentifier(nullDataTableName) + " WHERE ${CONDITIONS}");

    Extractor extractor = new GenericJdbcExtractor();
    DummyNullDataWriter writer = new DummyNullDataWriter();
    Schema schema = new Schema("TestExtractor");
    schema
        .addColumn(new FixedPoint("c1", 2L, true))
        .addColumn(new Decimal("c2", 5, 2))
        .addColumn(new Text("c3"))
        .addColumn(new Date("c4"));

    ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);

    GenericJdbcPartition partition = new GenericJdbcPartition();
    partition.setConditions("-50 <= ICOL AND ICOL < -16");
    extractor.extract(extractorContext, linkConfig, jobConfig, partition);
  }
  @SuppressWarnings({"unchecked", "rawtypes"})
  @Test
  public void testSubquery() throws Exception {
    MutableContext context = new MutableMapContext();

    LinkConfiguration linkConfig = new LinkConfiguration();

    linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
    linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;

    FromJobConfiguration jobConfig = new FromJobConfiguration();

    context.setString(
        GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL,
        "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL,SQOOP_SUBQUERY_ALIAS.DATECOL FROM "
            + "(SELECT * FROM "
            + executor.delimitIdentifier(tableName)
            + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS");

    GenericJdbcPartition partition;

    Extractor extractor = new GenericJdbcExtractor();
    DummyWriter writer = new DummyWriter();
    Schema schema = new Schema("TestExtractor");
    // dummy columns added, all we need is the column count to match to the
    // result set
    schema
        .addColumn(new FixedPoint("c1", 2L, true))
        .addColumn(new Text("c2"))
        .addColumn(new Date("c3"));

    ExtractorContext extractorContext = new ExtractorContext(context, writer, schema);

    partition = new GenericJdbcPartition();
    partition.setConditions("-50 <= ICOL AND ICOL < -16");
    extractor.extract(extractorContext, linkConfig, jobConfig, partition);

    partition = new GenericJdbcPartition();
    partition.setConditions("-16 <= ICOL AND ICOL < 17");
    extractor.extract(extractorContext, linkConfig, jobConfig, partition);

    partition = new GenericJdbcPartition();
    partition.setConditions("17 <= ICOL AND ICOL < 50");
    extractor.extract(extractorContext, linkConfig, jobConfig, partition);
  }
示例#4
0
  @SuppressWarnings({"unchecked", "rawtypes"})
  @Override
  public void run(Context context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    String extractorName = conf.get(MRJobConstants.JOB_ETL_EXTRACTOR);
    Extractor extractor = (Extractor) ClassUtils.instantiate(extractorName);

    Schema fromSchema = MRConfigurationUtils.getConnectorSchema(Direction.FROM, conf);
    Schema toSchema = MRConfigurationUtils.getConnectorSchema(Direction.TO, conf);
    matcher = MatcherFactory.getMatcher(fromSchema, toSchema);

    String fromIDFClass = conf.get(MRJobConstants.FROM_INTERMEDIATE_DATA_FORMAT);
    fromIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(fromIDFClass);
    fromIDF.setSchema(matcher.getFromSchema());
    String toIDFClass = conf.get(MRJobConstants.TO_INTERMEDIATE_DATA_FORMAT);
    toIDF = (IntermediateDataFormat<Object>) ClassUtils.instantiate(toIDFClass);
    toIDF.setSchema(matcher.getToSchema());

    // Objects that should be passed to the Executor execution
    PrefixContext subContext =
        new PrefixContext(conf, MRJobConstants.PREFIX_CONNECTOR_FROM_CONTEXT);
    Object fromConfig = MRConfigurationUtils.getConnectorLinkConfig(Direction.FROM, conf);
    Object fromJob = MRConfigurationUtils.getConnectorJobConfig(Direction.FROM, conf);

    SqoopSplit split = context.getCurrentKey();
    ExtractorContext extractorContext =
        new ExtractorContext(
            subContext, new SqoopDataWriter(context, fromIDF, toIDF, matcher), fromSchema);

    try {
      LOG.info("Running extractor class " + extractorName);
      extractor.extract(extractorContext, fromConfig, fromJob, split.getPartition());
      LOG.info("Extractor has finished");
      context.getCounter(SqoopCounters.ROWS_READ).increment(extractor.getRowsRead());
    } catch (Exception e) {
      throw new SqoopException(MRExecutionError.MAPRED_EXEC_0017, e);
    } finally {
      LOG.info("Stopping progress service");
    }
  }