private AbstractRunner generateRunner(PluginType pluginType) {
AbstractRunner newRunner = null;
TaskPluginCollector pluginCollector;
switch (pluginType) {
case READER:
newRunner =
LoadUtil.loadPluginRunner(
pluginType, this.taskConfig.getString(CoreConstant.JOB_READER_NAME));
newRunner.setJobConf(this.taskConfig.getConfiguration(CoreConstant.JOB_READER_PARAMETER));
pluginCollector =
ClassUtil.instantiate(
taskCollectorClass,
AbstractTaskPluginCollector.class,
configuration,
this.taskCommunication,
PluginType.READER);
((ReaderRunner) newRunner)
.setRecordSender(new BufferedRecordExchanger(this.channel, pluginCollector));
/** 设置taskPlugin的collector，用来处理脏数据和job/task通信 */
newRunner.setTaskPluginCollector(pluginCollector);
break;
case WRITER:
newRunner =
LoadUtil.loadPluginRunner(
pluginType, this.taskConfig.getString(CoreConstant.JOB_WRITER_NAME));
newRunner.setJobConf(this.taskConfig.getConfiguration(CoreConstant.JOB_WRITER_PARAMETER));
pluginCollector =
ClassUtil.instantiate(
taskCollectorClass,
AbstractTaskPluginCollector.class,
configuration,
this.taskCommunication,
PluginType.WRITER);
((WriterRunner) newRunner)
.setRecordReceiver(new BufferedRecordExchanger(this.channel, pluginCollector));
/** 设置taskPlugin的collector，用来处理脏数据和job/task通信 */
newRunner.setTaskPluginCollector(pluginCollector);
break;
default:
throw DataXException.asDataXException(
FrameworkErrorCode.ARGUMENT_ERROR, "Cant generateRunner for:" + pluginType);
}
newRunner.setTaskGroupId(taskGroupId);
newRunner.setTaskId(this.taskId);
newRunner.setRunnerCommunication(this.taskCommunication);
return newRunner;
}
public TaskExecutor(Configuration taskConf, int attemptCount) {
// 获取该taskExecutor的配置
this.taskConfig = taskConf;
Validate.isTrue(
null != this.taskConfig.getConfiguration(CoreConstant.JOB_READER)
&& null != this.taskConfig.getConfiguration(CoreConstant.JOB_WRITER),
"[reader|writer]的插件参数不能为空!");
// 得到taskId
this.taskId = this.taskConfig.getInt(CoreConstant.TASK_ID);
this.attemptCount = attemptCount;
/** 由taskId得到该taskExecutor的Communication 要传给readerRunner和writerRunner，同时要传给channel作统计用 */
this.taskCommunication = containerCommunicator.getCommunication(taskId);
Validate.notNull(
this.taskCommunication, String.format("taskId[%d]的Communication没有注册过", taskId));
this.channel = ClassUtil.instantiate(channelClazz, Channel.class, configuration);
this.channel.setCommunication(this.taskCommunication);
/** 生成writerThread */
writerRunner = (WriterRunner) generateRunner(PluginType.WRITER);
this.writerThread =
new Thread(
writerRunner, String.format("%d-%d-%d-writer", jobId, taskGroupId, this.taskId));
// 通过设置thread的contextClassLoader，即可实现同步和主程序不通的加载器
this.writerThread.setContextClassLoader(
LoadUtil.getJarLoader(
PluginType.WRITER, this.taskConfig.getString(CoreConstant.JOB_WRITER_NAME)));
/** 生成readerThread */
readerRunner = (ReaderRunner) generateRunner(PluginType.READER);
this.readerThread =
new Thread(
readerRunner, String.format("%d-%d-%d-reader", jobId, taskGroupId, this.taskId));
/** 通过设置thread的contextClassLoader，即可实现同步和主程序不通的加载器 */
this.readerThread.setContextClassLoader(
LoadUtil.getJarLoader(
PluginType.READER, this.taskConfig.getString(CoreConstant.JOB_READER_NAME)));
}
