Example #1
0
  public DBRecordReader(JDBCSplit split, DatabaseProperties dbProperties, DBManager dbManager) {
    DatabaseType databaseType = null;
    String sqlQuery = null;
    try {
      this.split = split;
      databaseProperties = dbProperties;
      connection = dbManager.getConnection();
      connection.setAutoCommit(false);
      statement =
          connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
      if (dbProperties.getConnectionUrl() != null) {
        String databaseName = dbProperties.getConnectionUrl().split(":")[1];
        databaseType = dbManager.getDatabaseType(databaseName);
      } else {
        databaseType = dbManager.getDatabaseName(connection);
      }
      sqlQuery =
          new QueryConstructor().constructSelectQueryForReading(dbProperties, split, databaseType);
      results = statement.executeQuery(sqlQuery);

    } catch (ClassNotFoundException e) {
      log.error("Failed to get connection", e);
    } catch (SQLException e) {
      log.error("Failed to fetch data from database using query: " + sqlQuery, e);
    } catch (UnsupportedDatabaseException e) {
      log.error("This database doesn't support by hive-jdbc-handler", e);
    }
  }
Example #2
0
  public boolean next(LongWritable key, MapWritable value) throws IOException {
    try {
      if (!results.next()) {
        return false;
      }

      // Set the key field value as the output key value
      key.set(pos + split.getStart());

      ResultSetMetaData resultsMetaData = results.getMetaData();
      int columnCount = resultsMetaData.getColumnCount();

      List<String> names = new ArrayList<String>();
      List<Integer> types = new ArrayList<Integer>();
      // The column count starts from 1
      for (int i = 1; i <= columnCount; i++) {
        // This is the column name in db table
        String name = resultsMetaData.getColumnName(i).toLowerCase();
        // Get the relevant metaTable name
        name = databaseProperties.getInputColumnMappingFields().get(name);
        int type = resultsMetaData.getColumnType(i);
        // Hive keeps column names in lowercase
        names.add(name.toLowerCase());
        types.add(type);
      }

      for (int j = 0; j < types.size(); j++) {
        value.put(new Text(names.get(j)), getActualObjectTypeForValue(results, types, j));
      }

      pos++;
    } catch (SQLException e) {
      throw new IOException(e.getMessage());
    }
    return true;
  }