Exemplo n.º 1
0
  private long putCachePatch(
      FileProtocol protocol, InputStream content, ImportBean bean, String user)
      throws BulkLoaderSystemException {
    assert protocol != null;
    assert content != null;
    assert bean != null;
    assert user != null;
    assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE
        || protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;

    CacheInfo info = protocol.getInfo();
    assert info != null;

    ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
    if (targetTableBean == null) {
      // 対応するテーブルの定義がDSL存在しない場合異常終了する。
      throw new BulkLoaderSystemException(
          getClass(),
          "TG-EXTRACTOR-02001",
          MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}", info.getTableName()));
    }

    URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
    try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) {
      LOG.info(
          "TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
      storage.putPatchCacheInfo(info);
      LOG.info(
          "TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());

      Class<?> targetTableModel = targetTableBean.getImportTargetType();
      Path targetUri = storage.getPatchContents("0");
      LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
      long recordCount = write(targetTableModel, targetUri.toUri(), content);
      LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
      LOG.info(
          "TG-PROFILE-01002",
          bean.getTargetName(),
          bean.getBatchId(),
          bean.getJobflowId(),
          bean.getExecutionId(),
          info.getTableName(),
          recordCount);
      return recordCount;
    } catch (IOException e) {
      throw new BulkLoaderSystemException(
          e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath);
    }
  }
Exemplo n.º 2
0
 private List<TestDataModel> collect(CacheStorage storage, Path contents) throws IOException {
   List<TestDataModel> results = new ArrayList<TestDataModel>();
   FileSystem fs = storage.getFileSystem();
   for (FileStatus status : fs.globStatus(contents)) {
     results.addAll(collectContent(fs, status));
   }
   Collections.sort(results);
   return results;
 }
Exemplo n.º 3
0
  /**
   * Creates a new cache with some deleted values.
   *
   * @throws Exception if failed
   */
  @Test
  public void create_deleted() throws Exception {
    CacheInfo info =
        new CacheInfo(
            "a",
            "id",
            calendar("2011-12-13 14:15:16"),
            "EXAMPLE",
            Collections.singleton("COL"),
            "com.example.Model",
            123L);
    framework.deployLibrary(TestDataModel.class, "batchapps/tbatch/lib/jobflow-tflow.jar");
    CacheStorage storage = new CacheStorage(getConfiguration(), getTargetUri());
    try {
      storage.putPatchCacheInfo(info);
      ModelOutput<TestDataModel> output = create(storage, storage.getPatchContents("0"));
      try {
        TestDataModel model = new TestDataModel();
        for (int i = 0; i < 100; i++) {
          model.systemId.set(i);
          model.deleted.set(i % 10 != 0);
          output.write(model);
        }
      } finally {
        output.close();
      }

      execute(CacheBuildClient.SUBCOMMAND_CREATE);
      assertThat(storage.getHeadCacheInfo(), is(info));

      List<TestDataModel> results = collect(storage, storage.getHeadContents("*"));
      assertThat(results.size(), is(10));
      for (int i = 0; i < 10; i++) {
        assertThat(results.get(i).systemId.get(), is(i * 10L));
      }
    } finally {
      storage.close();
    }
  }
Exemplo n.º 4
0
 private ModelOutput<TestDataModel> create(CacheStorage storage, Path path) throws IOException {
   return TemporaryStorage.openOutput(
       storage.getFileSystem().getConf(), TestDataModel.class, path);
 }
Exemplo n.º 5
0
  /**
   * Update a cache.
   *
   * @throws Exception if failed
   */
  @Test
  public void update_delete() throws Exception {
    CacheInfo info =
        new CacheInfo(
            "a",
            "id",
            calendar("2011-12-13 14:15:16"),
            "EXAMPLE",
            Collections.singleton("COL"),
            "com.example.Model",
            123L);
    framework.deployLibrary(TestDataModel.class, "batchapps/tbatch/lib/jobflow-tflow.jar");
    CacheStorage storage = new CacheStorage(getConfiguration(), getTargetUri());
    try {
      storage.putPatchCacheInfo(info);
      ModelOutput<TestDataModel> head = create(storage, storage.getHeadContents("0"));
      try {
        TestDataModel model = new TestDataModel();
        for (int i = 0; i < 10; i++) {
          model.systemId.set(i);
          model.value.set("HEAD");
          model.deleted.set(false);
          head.write(model);
        }
      } finally {
        head.close();
      }
      ModelOutput<TestDataModel> patch = create(storage, storage.getPatchContents("0"));
      try {
        TestDataModel model = new TestDataModel();
        for (int i = 0; i < 10; i += 2) {
          model.systemId.set(i);
          model.value.set("NEXT");
          model.deleted.set(i % 4 == 0);
          patch.write(model);
        }
      } finally {
        patch.close();
      }

      execute(CacheBuildClient.SUBCOMMAND_UPDATE);
      assertThat(storage.getHeadCacheInfo(), is(info));

      List<TestDataModel> results = collect(storage, storage.getHeadContents("*"));
      assertThat(results.size(), is(7));
      assertThat(results.get(0).systemId.get(), is(1L));
      assertThat(results.get(0).value.toString(), is("HEAD"));
      assertThat(results.get(1).systemId.get(), is(2L));
      assertThat(results.get(1).value.toString(), is("NEXT"));
      assertThat(results.get(2).systemId.get(), is(3L));
      assertThat(results.get(2).value.toString(), is("HEAD"));
      assertThat(results.get(3).systemId.get(), is(5L));
      assertThat(results.get(3).value.toString(), is("HEAD"));
      assertThat(results.get(4).systemId.get(), is(6L));
      assertThat(results.get(4).value.toString(), is("NEXT"));
      assertThat(results.get(5).systemId.get(), is(7L));
      assertThat(results.get(5).value.toString(), is("HEAD"));
      assertThat(results.get(6).systemId.get(), is(9L));
      assertThat(results.get(6).value.toString(), is("HEAD"));
    } finally {
      storage.close();
    }
  }