@Override
public FileOutput open(TaskSource taskSource, final FileOutput fileOutput) {
final PluginTask task = taskSource.loadTask(PluginTask.class);
final FileOutputOutputStream output =
new FileOutputOutputStream(
fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
return new OutputStreamFileOutput(
new OutputStreamFileOutput.Provider() {
public OutputStream openNext() throws IOException {
output.nextFile();
return new GZIPOutputStream(output) {
{
this.def.setLevel(task.getLevel());
}
};
}
public void finish() throws IOException {
fileOutput.finish();
}
public void close() throws IOException {
fileOutput.close();
}
});
}
@Override
public ConfigDiff resume(
TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control) {
PluginTask task = taskSource.loadTask(getTaskClass());
// TODO when parallel execution is implemented and enabled, (maybe) order_by
// is necessary to resume. transaction() gets the range of order_by
// colum and set it to WHERE condition to make the operation deterministic
return buildNextConfigDiff(task, control.run(taskSource, schema, taskCount));
}
@Override
public PageOutput open(
TaskSource taskSource,
final Schema inputSchema,
final Schema outputSchema,
final PageOutput output) {
final PluginTask task = taskSource.loadTask(PluginTask.class);
// create jsonColumns/baseColumn
final List<Column> outputColumns = outputSchema.getColumns();
final List<Column> inputColumns = inputSchema.getColumns();
Map<String, Column> inputColumnMap = Maps.newHashMap();
final List<Column> jsonColumns = new ArrayList<>();
for (Column column : outputColumns) {
if (!inputColumns.contains(column)) {
jsonColumns.add(column);
} else {
inputColumnMap.put(column.getName(), column);
}
}
final Column baseColumn = inputColumnMap.get(task.getBaseColumn().getName());
// create timestampParserMap
final HashMap<String, TimestampParser> timestampParserMap = Maps.newHashMap();
for (ColumnConfig jsonColumnConfig : task.getJsonColumns()) {
if (Types.TIMESTAMP.equals(jsonColumnConfig.getType())) {
String format = jsonColumnConfig.getOption().get(String.class, "format");
DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
timestampParserMap.put(task.getJoinedColumnsPrefix() + jsonColumnConfig.getName(), parser);
}
}
// get jsonTable
final HashMap<String, HashMap<String, String>> jsonTable = task.getJsonTable();
return new LeftOuterJoinJsonTableFilterFilteredPageOutput(
inputSchema, outputSchema, baseColumn, jsonTable, jsonColumns, timestampParserMap, output);
}
@Override
public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
PluginTask task = taskSource.loadTask(getTaskClass());
JdbcSchema querySchema = task.getQuerySchema();
BufferAllocator allocator = task.getBufferAllocator();
PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
try {
List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
try (JdbcInputConnection con = newConnection(task)) {
try (BatchSelect cursor =
con.newSelectCursor(
getQuery(task, con), task.getFetchRows(), task.getSocketTimeout())) {
while (true) {
// TODO run fetch() in another thread asynchronously
// TODO retry fetch() if it failed (maybe order_by is required and unique_column(s)
// option is also required)
boolean cont = fetch(cursor, getters, pageBuilder);
if (!cont) {
break;
}
}
}
}
} catch (SQLException ex) {
throw Throwables.propagate(ex);
}
pageBuilder.finish();
TaskReport report = Exec.newTaskReport();
// TODO
// if (orderByColumn != null) {
// report.set("last_value", lastValue);
// }
return report;
}