private long putCachePatch(
FileProtocol protocol, InputStream content, ImportBean bean, String user)
throws BulkLoaderSystemException {
assert protocol != null;
assert content != null;
assert bean != null;
assert user != null;
assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE
|| protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;
CacheInfo info = protocol.getInfo();
assert info != null;
ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
if (targetTableBean == null) {
// 対応するテーブルの定義がDSL存在しない場合異常終了する。
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-02001",
MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}", info.getTableName()));
}
URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) {
LOG.info(
"TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
storage.putPatchCacheInfo(info);
LOG.info(
"TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());
Class<?> targetTableModel = targetTableBean.getImportTargetType();
Path targetUri = storage.getPatchContents("0");
LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
long recordCount = write(targetTableModel, targetUri.toUri(), content);
LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
LOG.info(
"TG-PROFILE-01002",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
info.getTableName(),
recordCount);
return recordCount;
} catch (IOException e) {
throw new BulkLoaderSystemException(
e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath);
}
}
private List<TestDataModel> collect(CacheStorage storage, Path contents) throws IOException {
List<TestDataModel> results = new ArrayList<TestDataModel>();
FileSystem fs = storage.getFileSystem();
for (FileStatus status : fs.globStatus(contents)) {
results.addAll(collectContent(fs, status));
}
Collections.sort(results);
return results;
}
/**
* Creates a new cache with some deleted values.
*
* @throws Exception if failed
*/
@Test
public void create_deleted() throws Exception {
CacheInfo info =
new CacheInfo(
"a",
"id",
calendar("2011-12-13 14:15:16"),
"EXAMPLE",
Collections.singleton("COL"),
"com.example.Model",
123L);
framework.deployLibrary(TestDataModel.class, "batchapps/tbatch/lib/jobflow-tflow.jar");
CacheStorage storage = new CacheStorage(getConfiguration(), getTargetUri());
try {
storage.putPatchCacheInfo(info);
ModelOutput<TestDataModel> output = create(storage, storage.getPatchContents("0"));
try {
TestDataModel model = new TestDataModel();
for (int i = 0; i < 100; i++) {
model.systemId.set(i);
model.deleted.set(i % 10 != 0);
output.write(model);
}
} finally {
output.close();
}
execute(CacheBuildClient.SUBCOMMAND_CREATE);
assertThat(storage.getHeadCacheInfo(), is(info));
List<TestDataModel> results = collect(storage, storage.getHeadContents("*"));
assertThat(results.size(), is(10));
for (int i = 0; i < 10; i++) {
assertThat(results.get(i).systemId.get(), is(i * 10L));
}
} finally {
storage.close();
}
}
private ModelOutput<TestDataModel> create(CacheStorage storage, Path path) throws IOException {
return TemporaryStorage.openOutput(
storage.getFileSystem().getConf(), TestDataModel.class, path);
}
/**
* Update a cache.
*
* @throws Exception if failed
*/
@Test
public void update_delete() throws Exception {
CacheInfo info =
new CacheInfo(
"a",
"id",
calendar("2011-12-13 14:15:16"),
"EXAMPLE",
Collections.singleton("COL"),
"com.example.Model",
123L);
framework.deployLibrary(TestDataModel.class, "batchapps/tbatch/lib/jobflow-tflow.jar");
CacheStorage storage = new CacheStorage(getConfiguration(), getTargetUri());
try {
storage.putPatchCacheInfo(info);
ModelOutput<TestDataModel> head = create(storage, storage.getHeadContents("0"));
try {
TestDataModel model = new TestDataModel();
for (int i = 0; i < 10; i++) {
model.systemId.set(i);
model.value.set("HEAD");
model.deleted.set(false);
head.write(model);
}
} finally {
head.close();
}
ModelOutput<TestDataModel> patch = create(storage, storage.getPatchContents("0"));
try {
TestDataModel model = new TestDataModel();
for (int i = 0; i < 10; i += 2) {
model.systemId.set(i);
model.value.set("NEXT");
model.deleted.set(i % 4 == 0);
patch.write(model);
}
} finally {
patch.close();
}
execute(CacheBuildClient.SUBCOMMAND_UPDATE);
assertThat(storage.getHeadCacheInfo(), is(info));
List<TestDataModel> results = collect(storage, storage.getHeadContents("*"));
assertThat(results.size(), is(7));
assertThat(results.get(0).systemId.get(), is(1L));
assertThat(results.get(0).value.toString(), is("HEAD"));
assertThat(results.get(1).systemId.get(), is(2L));
assertThat(results.get(1).value.toString(), is("NEXT"));
assertThat(results.get(2).systemId.get(), is(3L));
assertThat(results.get(2).value.toString(), is("HEAD"));
assertThat(results.get(3).systemId.get(), is(5L));
assertThat(results.get(3).value.toString(), is("HEAD"));
assertThat(results.get(4).systemId.get(), is(6L));
assertThat(results.get(4).value.toString(), is("NEXT"));
assertThat(results.get(5).systemId.get(), is(7L));
assertThat(results.get(5).value.toString(), is("HEAD"));
assertThat(results.get(6).systemId.get(), is(9L));
assertThat(results.get(6).value.toString(), is("HEAD"));
} finally {
storage.close();
}
}
