private long putCachePatch(
FileProtocol protocol, InputStream content, ImportBean bean, String user)
throws BulkLoaderSystemException {
assert protocol != null;
assert content != null;
assert bean != null;
assert user != null;
assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE
|| protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;
CacheInfo info = protocol.getInfo();
assert info != null;
ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
if (targetTableBean == null) {
// 対応するテーブルの定義がDSL存在しない場合異常終了する。
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-02001",
MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}", info.getTableName()));
}
URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) {
LOG.info(
"TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
storage.putPatchCacheInfo(info);
LOG.info(
"TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());
Class<?> targetTableModel = targetTableBean.getImportTargetType();
Path targetUri = storage.getPatchContents("0");
LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
long recordCount = write(targetTableModel, targetUri.toUri(), content);
LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
LOG.info(
"TG-PROFILE-01002",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
info.getTableName(),
recordCount);
return recordCount;
} catch (IOException e) {
throw new BulkLoaderSystemException(
e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath);
}
}
/**
* Updates a cache (local information was broken).
* @throws Exception if failed
*/
@Test
public void update_cache_broken_local() throws Exception {
ImportBean bean = createBean();
Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();
final ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId("tb1");
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable.put("__TG_TEST1", tb1);
bean.setTargetTable(targetTable);
ImportProtocolDecide service = new ImportProtocolDecide() {
@Override
protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
throws BulkLoaderSystemException {
return Collections.singletonMap("tb1", new CacheInfo(
CacheInfo.FEATURE_VERSION,
tb1.getCacheId(),
offset(-1),
"__TG_TEST1",
tb1.getImportTargetColumns(),
tb1.getImportTargetType().getName(),
new ImportTarget1().__tgc__DataModelVersion()));
}
};
service.execute(bean);
assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
}
private void importContent(
FileProtocol protocol, InputStream content, ImportBean bean, String user)
throws BulkLoaderSystemException {
assert protocol != null;
assert content != null;
assert bean != null;
assert user != null;
String tableName = FileNameUtil.getImportTableName(protocol.getLocation());
ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName);
if (targetTableBean == null) {
// 対応するテーブルの定義がDSL存在しない場合異常終了する。
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-02001",
MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}", tableName));
}
URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath());
Class<?> targetTableModel = targetTableBean.getImportTargetType();
LOG.info("TG-EXTRACTOR-02002", tableName, dfsFilePath.toString(), targetTableModel.toString());
// ファイルをジョブ入力データ領域に書き出す
long recordCount = write(targetTableModel, dfsFilePath, content);
LOG.info("TG-EXTRACTOR-02003", tableName, dfsFilePath.toString(), targetTableModel.toString());
LOG.info(
"TG-PROFILE-01002",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
tableName,
recordCount);
}
/**
* Updates a cache (remote information was broken).
* @throws Exception if failed
*/
@Test
public void update_cache_broken_remote() throws Exception {
ImportBean bean = createBean();
Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();
final ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId("tb1");
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable.put("__TG_TEST1", tb1);
Connection conn = DBConnection.getConnection();
try {
LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
repo.putCacheInfo(new LocalCacheInfo(
tb1.getCacheId(),
null,
null,
"__TG_TEST1",
tb1.getDfsFilePath()));
} finally {
conn.close();
}
bean.setTargetTable(targetTable);
ImportProtocolDecide service = new ImportProtocolDecide() {
@Override
protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
throws BulkLoaderSystemException {
return Collections.emptyMap();
}
};
service.execute(bean);
assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
}
/**
* only normal contents.
* @throws Exception if failed
*/
@Test
public void contents() throws Exception {
ImportBean bean = createBean();
Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();
ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId(null);
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable.put("__TG_TEST1", tb1);
bean.setTargetTable(targetTable);
ImportProtocolDecide service = new ImportProtocolDecide();
service.execute(bean);
assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CONTENT));
assertThat(tb1.getStartTimestamp(), is(nullValue()));
}
/**
* Cache lock is conflict.
* @throws Exception if failed
*/
@Test
public void release_lock() throws Exception {
ImportBean bean1 = createBean();
Map<String, ImportTargetTableBean> targetTable1 = new HashMap<String, ImportTargetTableBean>();
ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId("tb1");
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable1.put("__TG_TEST1", tb1);
bean1.setTargetTable(targetTable1);
ImportBean bean2 = createBean();
Map<String, ImportTargetTableBean> targetTable2 = new HashMap<String, ImportTargetTableBean>();
ImportTargetTableBean tb2 = new ImportTargetTableBean();
tb2.setCacheId("tb1");
tb2.setDfsFilePath("tb1");
tb2.setImportTargetType(ImportTarget1.class);
tb2.setImportTargetColumns(Arrays.asList("A"));
tb2.setSearchCondition("");
targetTable2.put("__TG_TEST1", tb2);
bean2.setTargetTable(targetTable2);
ImportProtocolDecide service = new ImportProtocolDecide() {
@Override
protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
throws BulkLoaderSystemException {
return Collections.emptyMap();
}
};
service.execute(bean1);
service.cleanUpForRetry(bean1);
service.execute(bean2);
// ok.
}
/**
* Updates a cache.
* @throws Exception if failed
*/
@Test
public void update_cache_rebuild() throws Exception {
ImportBean bean = createBean();
Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();
final ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId("tb1");
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable.put("__TG_TEST1", tb1);
Connection conn = DBConnection.getConnection();
final Calendar last = offset(-1);
try {
LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn);
repo.putCacheInfo(new LocalCacheInfo(
tb1.getCacheId(),
null,
last,
"__TG_TEST1",
tb1.getDfsFilePath()));
} finally {
conn.close();
}
bean.setTargetTable(targetTable);
ImportProtocolDecide service = new ImportProtocolDecide() {
@Override
protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
throws BulkLoaderSystemException {
return Collections.singletonMap("tb1", new CacheInfo(
CacheInfo.FEATURE_VERSION,
tb1.getCacheId(),
last,
"__TG_TEST1",
tb1.getImportTargetColumns(),
tb1.getImportTargetType().getName(),
new ImportTarget1().__tgc__DataModelVersion()));
}
};
service.execute(bean);
assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.UPDATE_CACHE));
assertThat(tb1.getImportProtocol().getLocation(), is(tb1.getDfsFilePath()));
assertThat(tb1.getStartTimestamp(), is(notNullValue()));
CacheInfo info = tb1.getImportProtocol().getInfo();
assertThat(info, is(notNullValue()));
assertThat(info.getId(), is("tb1"));
assertThat(info.getFeatureVersion(), is(CacheInfo.FEATURE_VERSION));
assertThat(info.getTimestamp(), is(not(nullValue())));
assertThat(info.getTableName(), is("__TG_TEST1"));
assertThat(info.getColumnNames(), is((Object) new HashSet<String>(tb1.getImportTargetColumns())));
assertThat(info.getModelClassName(), is(ImportTarget1.class.getName()));
assertThat(info.getModelClassVersion(), is(new ImportTarget1().__tgc__DataModelVersion()));
}
/**
* Creates a new cache.
* @throws Exception if failed
*/
@Test
public void create_cache() throws Exception {
ImportBean bean = createBean();
Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>();
ImportTargetTableBean tb1 = new ImportTargetTableBean();
tb1.setCacheId("tb1");
tb1.setDfsFilePath("tb1");
tb1.setImportTargetType(ImportTarget1.class);
tb1.setImportTargetColumns(Arrays.asList("A"));
tb1.setSearchCondition("");
targetTable.put("__TG_TEST1", tb1);
bean.setTargetTable(targetTable);
ImportProtocolDecide service = new ImportProtocolDecide() {
@Override
protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _)
throws BulkLoaderSystemException {
return Collections.emptyMap();
}
};
service.execute(bean);
assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE));
assertThat(tb1.getImportProtocol().getLocation(), is(tb1.getDfsFilePath()));
assertThat(tb1.getStartTimestamp(), is(nullValue()));
CacheInfo info = tb1.getImportProtocol().getInfo();
assertThat(info, is(notNullValue()));
assertThat(info.getId(), is("tb1"));
assertThat(info.getFeatureVersion(), is(CacheInfo.FEATURE_VERSION));
assertThat(info.getTimestamp(), is(not(nullValue())));
assertThat(info.getTableName(), is("__TG_TEST1"));
assertThat(info.getColumnNames(), is((Object) new HashSet<String>(tb1.getImportTargetColumns())));
assertThat(info.getModelClassName(), is(ImportTarget1.class.getName()));
assertThat(info.getModelClassVersion(), is(new ImportTarget1().__tgc__DataModelVersion()));
}