예제 #1
0
  /**
   * 標準入力を読み込んでDFSにファイルを書き出す。 {@link FileList}形式で受け取ったTSV形式のファイルをModelオブジェクトに変換して、テンポラリ入力データとして配置する。
   * 出力先はプロトコルの形式によって異なる。 利用可能なプロトコルは以下のとおり。
   *
   * <ul>
   *   <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#CONTENT}
   *   <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#CREATE_CACHE}
   *   <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#UPDATE_CACHE}
   * </ul>
   *
   * @param bean パラメータを保持するBean
   * @param user OSのユーザー名
   * @return 出力結果(true:正常終了、false:異常終了)
   */
  public boolean importFile(ImportBean bean, String user) {
    // 標準入力を取得
    FileList.Reader reader;
    try {
      reader = FileList.createReader(getInputStream());
    } catch (IOException e) {
      LOG.error(e, "TG-EXTRACTOR-02001", "標準入力からFileListの取得に失敗");
      return false;
    }
    try {
      // FileListの終端まで繰り返す
      List<Future<?>> running = new ArrayList<>();
      while (reader.next()) {
        FileProtocol protocol = reader.getCurrentProtocol();
        try (InputStream content = reader.openContent()) {
          switch (protocol.getKind()) {
            case CONTENT:
              importContent(protocol, content, bean, user);
              break;

            case CREATE_CACHE:
            case UPDATE_CACHE:
              long recordCount = putCachePatch(protocol, content, bean, user);
              Callable<?> builder = createCacheBuilder(protocol, bean, user, recordCount);
              if (builder != null) {
                LOG.debugMessage(
                    "Submitting cache builder: {0} {1}",
                    protocol.getKind(), protocol.getInfo().getTableName());
                running.add(executor.submit(builder));
              }
              break;

            default:
              throw new AssertionError(protocol.getKind());
          }
        }
      }

      waitForCompleteTasks(bean, running);
      // 正常終了
      return true;

    } catch (BulkLoaderSystemException e) {
      LOG.log(e);
    } catch (IOException e) {
      // FileListの展開に失敗
      LOG.error(e, "TG-EXTRACTOR-02001", "標準入力からFileListの取得に失敗");
    } finally {
      try {
        reader.close();
      } catch (IOException e) {
        // ここで例外が発生した場合は握りつぶす
        e.printStackTrace();
      }
    }
    return false;
  }
예제 #2
0
  private long putCachePatch(
      FileProtocol protocol, InputStream content, ImportBean bean, String user)
      throws BulkLoaderSystemException {
    assert protocol != null;
    assert content != null;
    assert bean != null;
    assert user != null;
    assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE
        || protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;

    CacheInfo info = protocol.getInfo();
    assert info != null;

    ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
    if (targetTableBean == null) {
      // 対応するテーブルの定義がDSL存在しない場合異常終了する。
      throw new BulkLoaderSystemException(
          getClass(),
          "TG-EXTRACTOR-02001",
          MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}", info.getTableName()));
    }

    URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
    try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) {
      LOG.info(
          "TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
      storage.putPatchCacheInfo(info);
      LOG.info(
          "TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());

      Class<?> targetTableModel = targetTableBean.getImportTargetType();
      Path targetUri = storage.getPatchContents("0");
      LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
      long recordCount = write(targetTableModel, targetUri.toUri(), content);
      LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
      LOG.info(
          "TG-PROFILE-01002",
          bean.getTargetName(),
          bean.getBatchId(),
          bean.getJobflowId(),
          bean.getExecutionId(),
          info.getTableName(),
          recordCount);
      return recordCount;
    } catch (IOException e) {
      throw new BulkLoaderSystemException(
          e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath);
    }
  }
예제 #3
0
 private Callable<?> createCacheBuilder(
     FileProtocol protocol, ImportBean bean, String user, long recordCount)
     throws BulkLoaderSystemException {
   assert protocol != null;
   assert bean != null;
   assert user != null;
   CacheInfo info = protocol.getInfo();
   URI location = resolveLocation(bean, user, protocol.getLocation());
   assert info != null;
   try {
     switch (protocol.getKind()) {
       case CREATE_CACHE:
         return createCacheBuilder(CacheBuildClient.SUBCOMMAND_CREATE, bean, location, info);
       case UPDATE_CACHE:
         if (recordCount > 0) {
           return createCacheBuilder(CacheBuildClient.SUBCOMMAND_UPDATE, bean, location, info);
         } else {
           return null;
         }
       default:
         throw new AssertionError(protocol);
     }
   } catch (IOException e) {
     throw new BulkLoaderSystemException(
         e,
         getClass(),
         "TG-EXTRACTOR-12002",
         protocol.getKind(),
         info.getId(),
         info.getTableName(),
         bean.getTargetName(),
         bean.getBatchId(),
         bean.getJobflowId(),
         bean.getExecutionId());
   }
 }
예제 #4
0
  private void importContent(
      FileProtocol protocol, InputStream content, ImportBean bean, String user)
      throws BulkLoaderSystemException {
    assert protocol != null;
    assert content != null;
    assert bean != null;
    assert user != null;
    String tableName = FileNameUtil.getImportTableName(protocol.getLocation());

    ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName);
    if (targetTableBean == null) {
      // 対応するテーブルの定義がDSL存在しない場合異常終了する。
      throw new BulkLoaderSystemException(
          getClass(),
          "TG-EXTRACTOR-02001",
          MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}", tableName));
    }

    URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath());
    Class<?> targetTableModel = targetTableBean.getImportTargetType();

    LOG.info("TG-EXTRACTOR-02002", tableName, dfsFilePath.toString(), targetTableModel.toString());

    // ファイルをジョブ入力データ領域に書き出す
    long recordCount = write(targetTableModel, dfsFilePath, content);

    LOG.info("TG-EXTRACTOR-02003", tableName, dfsFilePath.toString(), targetTableModel.toString());
    LOG.info(
        "TG-PROFILE-01002",
        bean.getTargetName(),
        bean.getBatchId(),
        bean.getJobflowId(),
        bean.getExecutionId(),
        tableName,
        recordCount);
  }