Esempio n. 1
0
  @Override
  public RecordReader<LongWritable, MapWritable> getRecordReader(
      InputSplit split, JobConf conf, Reporter reporter) throws IOException {
    List<Integer> readColIDs = getReadColumnIDs(conf);

    boolean addAll = (readColIDs.size() == 0);

    String columnString = conf.get(ConfigurationUtil.COLUMN_MAPPING);
    if (StringUtils.isBlank(columnString)) {
      throw new IOException("no column mapping found!");
    }

    String[] columns = ConfigurationUtil.getAllColumns(columnString);
    if (readColIDs.size() > columns.length) {
      throw new IOException("read column count larger than that in column mapping string!");
    }

    String[] cols;
    if (addAll) {
      cols = columns;
    } else {
      cols = new String[readColIDs.size()];
      for (int i = 0; i < cols.length; i++) {
        cols[i] = columns[readColIDs.get(i)];
      }
    }
    String filterExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);

    if (filterExprSerialized != null) {
      ExprNodeDesc filterExpr = Utilities.deserializeExpression(filterExprSerialized, conf);
      /*String columnNameProperty = conf.get(
                  org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS);
      System.err.println("======list columns:" + columnNameProperty);*/
      dumpFilterExpr(filterExpr);
      // TODO:
    }

    return new SolrReader(
        ConfigurationUtil.getUrl(conf),
        (SolrSplit) split,
        cols,
        ConfigurationUtil.getNumInputBufferRows(conf));
  }
Esempio n. 2
0
  @Override
  public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {

    LOG.debug("Entry SdbSerDe::initialize");

    final String columnString = tbl.getProperty(ConfigurationUtil.COLUMN_MAPPING);
    if (StringUtils.isBlank(columnString)) {
      throw new SerDeException("No column mapping found, use " + ConfigurationUtil.COLUMN_MAPPING);
    }
    final String[] columnNamesArray = ConfigurationUtil.getAllColumns(columnString);
    fieldCount = columnNamesArray.length;
    columnNames = new ArrayList<String>(columnNamesArray.length);
    columnNames.addAll(Arrays.asList(columnNamesArray));

    serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, getClass().getName());

    LOG.debug(
        "EscapeChar:"
            + serdeParams.getEscapeChar()
            + "getSeparators:"
            + new String(serdeParams.getSeparators()));

    byte[] sparator = new byte[1];
    sparator[0] = '|';
    objectInspector =
        LazyFactory.createLazyStructInspector(
            serdeParams.getColumnNames(),
            serdeParams.getColumnTypes(),
            sparator,
            serdeParams.getNullSequence(),
            serdeParams.isLastColumnTakesRest(),
            serdeParams.isEscaped(),
            serdeParams.getEscapeChar());

    row = new LazyStruct((LazySimpleStructObjectInspector) objectInspector);

    LOG.debug("Exit SdbSerDe::initialize");
  }