コード例 #1
0
  @Override
  public FileOutput open(TaskSource taskSource, final FileOutput fileOutput) {
    final PluginTask task = taskSource.loadTask(PluginTask.class);

    final FileOutputOutputStream output =
        new FileOutputOutputStream(
            fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);

    return new OutputStreamFileOutput(
        new OutputStreamFileOutput.Provider() {
          public OutputStream openNext() throws IOException {
            output.nextFile();
            return new GZIPOutputStream(output) {
              {
                this.def.setLevel(task.getLevel());
              }
            };
          }

          public void finish() throws IOException {
            fileOutput.finish();
          }

          public void close() throws IOException {
            fileOutput.close();
          }
        });
  }
コード例 #2
0
  @Override
  public ConfigDiff resume(
      TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control) {
    PluginTask task = taskSource.loadTask(getTaskClass());

    // TODO when parallel execution is implemented and enabled, (maybe) order_by
    //      is necessary to resume. transaction() gets the range of order_by
    //      colum and set it to WHERE condition to make the operation deterministic

    return buildNextConfigDiff(task, control.run(taskSource, schema, taskCount));
  }
  @Override
  public PageOutput open(
      TaskSource taskSource,
      final Schema inputSchema,
      final Schema outputSchema,
      final PageOutput output) {
    final PluginTask task = taskSource.loadTask(PluginTask.class);

    // create jsonColumns/baseColumn
    final List<Column> outputColumns = outputSchema.getColumns();
    final List<Column> inputColumns = inputSchema.getColumns();

    Map<String, Column> inputColumnMap = Maps.newHashMap();
    final List<Column> jsonColumns = new ArrayList<>();
    for (Column column : outputColumns) {
      if (!inputColumns.contains(column)) {
        jsonColumns.add(column);
      } else {
        inputColumnMap.put(column.getName(), column);
      }
    }

    final Column baseColumn = inputColumnMap.get(task.getBaseColumn().getName());

    // create timestampParserMap
    final HashMap<String, TimestampParser> timestampParserMap = Maps.newHashMap();
    for (ColumnConfig jsonColumnConfig : task.getJsonColumns()) {
      if (Types.TIMESTAMP.equals(jsonColumnConfig.getType())) {
        String format = jsonColumnConfig.getOption().get(String.class, "format");
        DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
        TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
        timestampParserMap.put(task.getJoinedColumnsPrefix() + jsonColumnConfig.getName(), parser);
      }
    }

    // get jsonTable
    final HashMap<String, HashMap<String, String>> jsonTable = task.getJsonTable();

    return new LeftOuterJoinJsonTableFilterFilteredPageOutput(
        inputSchema, outputSchema, baseColumn, jsonTable, jsonColumns, timestampParserMap, output);
  }
コード例 #4
0
  @Override
  public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
    PluginTask task = taskSource.loadTask(getTaskClass());

    JdbcSchema querySchema = task.getQuerySchema();
    BufferAllocator allocator = task.getBufferAllocator();
    PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);

    try {
      List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);

      try (JdbcInputConnection con = newConnection(task)) {
        try (BatchSelect cursor =
            con.newSelectCursor(
                getQuery(task, con), task.getFetchRows(), task.getSocketTimeout())) {
          while (true) {
            // TODO run fetch() in another thread asynchronously
            // TODO retry fetch() if it failed (maybe order_by is required and unique_column(s)
            // option is also required)
            boolean cont = fetch(cursor, getters, pageBuilder);
            if (!cont) {
              break;
            }
          }
        }
      }

    } catch (SQLException ex) {
      throw Throwables.propagate(ex);
    }
    pageBuilder.finish();

    TaskReport report = Exec.newTaskReport();
    // TODO
    // if (orderByColumn != null) {
    //    report.set("last_value", lastValue);
    // }
    return report;
  }