コード例 #1
0
  @Override
  public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
    PluginTask task = taskSource.loadTask(getTaskClass());

    JdbcSchema querySchema = task.getQuerySchema();
    BufferAllocator allocator = task.getBufferAllocator();
    PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);

    try {
      List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);

      try (JdbcInputConnection con = newConnection(task)) {
        try (BatchSelect cursor =
            con.newSelectCursor(
                getQuery(task, con), task.getFetchRows(), task.getSocketTimeout())) {
          while (true) {
            // TODO run fetch() in another thread asynchronously
            // TODO retry fetch() if it failed (maybe order_by is required and unique_column(s)
            // option is also required)
            boolean cont = fetch(cursor, getters, pageBuilder);
            if (!cont) {
              break;
            }
          }
        }
      }

    } catch (SQLException ex) {
      throw Throwables.propagate(ex);
    }
    pageBuilder.finish();

    TaskReport report = Exec.newTaskReport();
    // TODO
    // if (orderByColumn != null) {
    //    report.set("last_value", lastValue);
    // }
    return report;
  }
コード例 #2
0
  private boolean fetch(BatchSelect cursor, List<ColumnGetter> getters, PageBuilder pageBuilder)
      throws SQLException {
    ResultSet result = cursor.fetch();
    if (result == null || !result.next()) {
      return false;
    }

    List<Column> columns = pageBuilder.getSchema().getColumns();
    long rows = 0;
    long reportRows = 500;
    do {
      for (int i = 0; i < getters.size(); i++) {
        int index = i + 1; // JDBC column index begins from 1
        getters.get(i).getAndSet(result, index, columns.get(i));
      }
      pageBuilder.addRecord();
      rows++;
      if (rows % reportRows == 0) {
        logger.info(String.format("Fetched %,d rows.", rows));
        reportRows *= 2;
      }
    } while (result.next());
    return true;
  }
コード例 #3
0
ファイル: MockParserPlugin.java プロジェクト: zoetrope/embulk
 @Override
 public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output) {
   try (final PageBuilder pageBuilder =
       new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
     while (input.nextFile()) {
       Buffer buffer = input.poll();
       if (buffer != null) {
         for (Column column : schema.getColumns()) {
           Type type = column.getType();
           switch (type.getName()) {
             case "boolean":
               pageBuilder.setBoolean(column, true);
               break;
             case "long":
               pageBuilder.setLong(column, 2L);
               break;
             case "double":
               pageBuilder.setDouble(column, 3.0D);
               break;
             case "string":
               pageBuilder.setString(column, "45");
               break;
             case "timestamp":
               pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(678L));
               break;
             default:
               throw new IllegalStateException("Unknown type: " + type.getName());
           }
         }
         pageBuilder.addRecord();
         if (raiseException) {
           throw new RuntimeException("emulated exception");
         }
       }
     }
     pageBuilder.finish();
   }
 }