Exemplo n.º 1
0
    private AbstractRunner generateRunner(PluginType pluginType) {
      AbstractRunner newRunner = null;
      TaskPluginCollector pluginCollector;

      switch (pluginType) {
        case READER:
          newRunner =
              LoadUtil.loadPluginRunner(
                  pluginType, this.taskConfig.getString(CoreConstant.JOB_READER_NAME));
          newRunner.setJobConf(this.taskConfig.getConfiguration(CoreConstant.JOB_READER_PARAMETER));

          pluginCollector =
              ClassUtil.instantiate(
                  taskCollectorClass,
                  AbstractTaskPluginCollector.class,
                  configuration,
                  this.taskCommunication,
                  PluginType.READER);

          ((ReaderRunner) newRunner)
              .setRecordSender(new BufferedRecordExchanger(this.channel, pluginCollector));
          /** 设置taskPlugin的collector,用来处理脏数据和job/task通信 */
          newRunner.setTaskPluginCollector(pluginCollector);
          break;
        case WRITER:
          newRunner =
              LoadUtil.loadPluginRunner(
                  pluginType, this.taskConfig.getString(CoreConstant.JOB_WRITER_NAME));
          newRunner.setJobConf(this.taskConfig.getConfiguration(CoreConstant.JOB_WRITER_PARAMETER));

          pluginCollector =
              ClassUtil.instantiate(
                  taskCollectorClass,
                  AbstractTaskPluginCollector.class,
                  configuration,
                  this.taskCommunication,
                  PluginType.WRITER);
          ((WriterRunner) newRunner)
              .setRecordReceiver(new BufferedRecordExchanger(this.channel, pluginCollector));
          /** 设置taskPlugin的collector,用来处理脏数据和job/task通信 */
          newRunner.setTaskPluginCollector(pluginCollector);
          break;
        default:
          throw DataXException.asDataXException(
              FrameworkErrorCode.ARGUMENT_ERROR, "Cant generateRunner for:" + pluginType);
      }

      newRunner.setTaskGroupId(taskGroupId);
      newRunner.setTaskId(this.taskId);
      newRunner.setRunnerCommunication(this.taskCommunication);

      return newRunner;
    }
Exemplo n.º 2
0
    public TaskExecutor(Configuration taskConf, int attemptCount) {
      // 获取该taskExecutor的配置
      this.taskConfig = taskConf;
      Validate.isTrue(
          null != this.taskConfig.getConfiguration(CoreConstant.JOB_READER)
              && null != this.taskConfig.getConfiguration(CoreConstant.JOB_WRITER),
          "[reader|writer]的插件参数不能为空!");

      // 得到taskId
      this.taskId = this.taskConfig.getInt(CoreConstant.TASK_ID);
      this.attemptCount = attemptCount;

      /** 由taskId得到该taskExecutor的Communication 要传给readerRunner和writerRunner,同时要传给channel作统计用 */
      this.taskCommunication = containerCommunicator.getCommunication(taskId);
      Validate.notNull(
          this.taskCommunication, String.format("taskId[%d]的Communication没有注册过", taskId));
      this.channel = ClassUtil.instantiate(channelClazz, Channel.class, configuration);
      this.channel.setCommunication(this.taskCommunication);

      /** 生成writerThread */
      writerRunner = (WriterRunner) generateRunner(PluginType.WRITER);
      this.writerThread =
          new Thread(
              writerRunner, String.format("%d-%d-%d-writer", jobId, taskGroupId, this.taskId));
      // 通过设置thread的contextClassLoader,即可实现同步和主程序不通的加载器
      this.writerThread.setContextClassLoader(
          LoadUtil.getJarLoader(
              PluginType.WRITER, this.taskConfig.getString(CoreConstant.JOB_WRITER_NAME)));

      /** 生成readerThread */
      readerRunner = (ReaderRunner) generateRunner(PluginType.READER);
      this.readerThread =
          new Thread(
              readerRunner, String.format("%d-%d-%d-reader", jobId, taskGroupId, this.taskId));
      /** 通过设置thread的contextClassLoader,即可实现同步和主程序不通的加载器 */
      this.readerThread.setContextClassLoader(
          LoadUtil.getJarLoader(
              PluginType.READER, this.taskConfig.getString(CoreConstant.JOB_READER_NAME)));
    }