private long putCachePatch( FileProtocol protocol, InputStream content, ImportBean bean, String user) throws BulkLoaderSystemException { assert protocol != null; assert content != null; assert bean != null; assert user != null; assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE || protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE; CacheInfo info = protocol.getInfo(); assert info != null; ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName()); if (targetTableBean == null) { // 対応するテーブルの定義がDSL存在しない場合異常終了する。 throw new BulkLoaderSystemException( getClass(), "TG-EXTRACTOR-02001", MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}", info.getTableName())); } URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation()); try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) { LOG.info( "TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties()); storage.putPatchCacheInfo(info); LOG.info( "TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties()); Class<?> targetTableModel = targetTableBean.getImportTargetType(); Path targetUri = storage.getPatchContents("0"); LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri); long recordCount = write(targetTableModel, targetUri.toUri(), content); LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount); LOG.info( "TG-PROFILE-01002", bean.getTargetName(), bean.getBatchId(), bean.getJobflowId(), bean.getExecutionId(), info.getTableName(), recordCount); return recordCount; } catch (IOException e) { throw new BulkLoaderSystemException( e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath); } }
/** * Updates a cache (local information was broken). * @throws Exception if failed */ @Test public void update_cache_broken_local() throws Exception { ImportBean bean = createBean(); Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>(); final ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId("tb1"); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable.put("__TG_TEST1", tb1); bean.setTargetTable(targetTable); ImportProtocolDecide service = new ImportProtocolDecide() { @Override protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _) throws BulkLoaderSystemException { return Collections.singletonMap("tb1", new CacheInfo( CacheInfo.FEATURE_VERSION, tb1.getCacheId(), offset(-1), "__TG_TEST1", tb1.getImportTargetColumns(), tb1.getImportTargetType().getName(), new ImportTarget1().__tgc__DataModelVersion())); } }; service.execute(bean); assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE)); }
private void importContent( FileProtocol protocol, InputStream content, ImportBean bean, String user) throws BulkLoaderSystemException { assert protocol != null; assert content != null; assert bean != null; assert user != null; String tableName = FileNameUtil.getImportTableName(protocol.getLocation()); ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName); if (targetTableBean == null) { // 対応するテーブルの定義がDSL存在しない場合異常終了する。 throw new BulkLoaderSystemException( getClass(), "TG-EXTRACTOR-02001", MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名:{0}", tableName)); } URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath()); Class<?> targetTableModel = targetTableBean.getImportTargetType(); LOG.info("TG-EXTRACTOR-02002", tableName, dfsFilePath.toString(), targetTableModel.toString()); // ファイルをジョブ入力データ領域に書き出す long recordCount = write(targetTableModel, dfsFilePath, content); LOG.info("TG-EXTRACTOR-02003", tableName, dfsFilePath.toString(), targetTableModel.toString()); LOG.info( "TG-PROFILE-01002", bean.getTargetName(), bean.getBatchId(), bean.getJobflowId(), bean.getExecutionId(), tableName, recordCount); }
/** * Updates a cache (remote information was broken). * @throws Exception if failed */ @Test public void update_cache_broken_remote() throws Exception { ImportBean bean = createBean(); Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>(); final ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId("tb1"); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable.put("__TG_TEST1", tb1); Connection conn = DBConnection.getConnection(); try { LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn); repo.putCacheInfo(new LocalCacheInfo( tb1.getCacheId(), null, null, "__TG_TEST1", tb1.getDfsFilePath())); } finally { conn.close(); } bean.setTargetTable(targetTable); ImportProtocolDecide service = new ImportProtocolDecide() { @Override protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _) throws BulkLoaderSystemException { return Collections.emptyMap(); } }; service.execute(bean); assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE)); }
/** * only normal contents. * @throws Exception if failed */ @Test public void contents() throws Exception { ImportBean bean = createBean(); Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>(); ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId(null); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable.put("__TG_TEST1", tb1); bean.setTargetTable(targetTable); ImportProtocolDecide service = new ImportProtocolDecide(); service.execute(bean); assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CONTENT)); assertThat(tb1.getStartTimestamp(), is(nullValue())); }
/** * Cache lock is conflict. * @throws Exception if failed */ @Test public void release_lock() throws Exception { ImportBean bean1 = createBean(); Map<String, ImportTargetTableBean> targetTable1 = new HashMap<String, ImportTargetTableBean>(); ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId("tb1"); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable1.put("__TG_TEST1", tb1); bean1.setTargetTable(targetTable1); ImportBean bean2 = createBean(); Map<String, ImportTargetTableBean> targetTable2 = new HashMap<String, ImportTargetTableBean>(); ImportTargetTableBean tb2 = new ImportTargetTableBean(); tb2.setCacheId("tb1"); tb2.setDfsFilePath("tb1"); tb2.setImportTargetType(ImportTarget1.class); tb2.setImportTargetColumns(Arrays.asList("A")); tb2.setSearchCondition(""); targetTable2.put("__TG_TEST1", tb2); bean2.setTargetTable(targetTable2); ImportProtocolDecide service = new ImportProtocolDecide() { @Override protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _) throws BulkLoaderSystemException { return Collections.emptyMap(); } }; service.execute(bean1); service.cleanUpForRetry(bean1); service.execute(bean2); // ok. }
/** * Updates a cache. * @throws Exception if failed */ @Test public void update_cache_rebuild() throws Exception { ImportBean bean = createBean(); Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>(); final ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId("tb1"); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable.put("__TG_TEST1", tb1); Connection conn = DBConnection.getConnection(); final Calendar last = offset(-1); try { LocalCacheInfoRepository repo = new LocalCacheInfoRepository(conn); repo.putCacheInfo(new LocalCacheInfo( tb1.getCacheId(), null, last, "__TG_TEST1", tb1.getDfsFilePath())); } finally { conn.close(); } bean.setTargetTable(targetTable); ImportProtocolDecide service = new ImportProtocolDecide() { @Override protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _) throws BulkLoaderSystemException { return Collections.singletonMap("tb1", new CacheInfo( CacheInfo.FEATURE_VERSION, tb1.getCacheId(), last, "__TG_TEST1", tb1.getImportTargetColumns(), tb1.getImportTargetType().getName(), new ImportTarget1().__tgc__DataModelVersion())); } }; service.execute(bean); assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.UPDATE_CACHE)); assertThat(tb1.getImportProtocol().getLocation(), is(tb1.getDfsFilePath())); assertThat(tb1.getStartTimestamp(), is(notNullValue())); CacheInfo info = tb1.getImportProtocol().getInfo(); assertThat(info, is(notNullValue())); assertThat(info.getId(), is("tb1")); assertThat(info.getFeatureVersion(), is(CacheInfo.FEATURE_VERSION)); assertThat(info.getTimestamp(), is(not(nullValue()))); assertThat(info.getTableName(), is("__TG_TEST1")); assertThat(info.getColumnNames(), is((Object) new HashSet<String>(tb1.getImportTargetColumns()))); assertThat(info.getModelClassName(), is(ImportTarget1.class.getName())); assertThat(info.getModelClassVersion(), is(new ImportTarget1().__tgc__DataModelVersion())); }
/** * Creates a new cache. * @throws Exception if failed */ @Test public void create_cache() throws Exception { ImportBean bean = createBean(); Map<String, ImportTargetTableBean> targetTable = new HashMap<String, ImportTargetTableBean>(); ImportTargetTableBean tb1 = new ImportTargetTableBean(); tb1.setCacheId("tb1"); tb1.setDfsFilePath("tb1"); tb1.setImportTargetType(ImportTarget1.class); tb1.setImportTargetColumns(Arrays.asList("A")); tb1.setSearchCondition(""); targetTable.put("__TG_TEST1", tb1); bean.setTargetTable(targetTable); ImportProtocolDecide service = new ImportProtocolDecide() { @Override protected Map<String, CacheInfo> collectRemoteCacheInfo(ImportBean _) throws BulkLoaderSystemException { return Collections.emptyMap(); } }; service.execute(bean); assertThat(tb1.getImportProtocol().getKind(), is(FileProtocol.Kind.CREATE_CACHE)); assertThat(tb1.getImportProtocol().getLocation(), is(tb1.getDfsFilePath())); assertThat(tb1.getStartTimestamp(), is(nullValue())); CacheInfo info = tb1.getImportProtocol().getInfo(); assertThat(info, is(notNullValue())); assertThat(info.getId(), is("tb1")); assertThat(info.getFeatureVersion(), is(CacheInfo.FEATURE_VERSION)); assertThat(info.getTimestamp(), is(not(nullValue()))); assertThat(info.getTableName(), is("__TG_TEST1")); assertThat(info.getColumnNames(), is((Object) new HashSet<String>(tb1.getImportTargetColumns()))); assertThat(info.getModelClassName(), is(ImportTarget1.class.getName())); assertThat(info.getModelClassVersion(), is(new ImportTarget1().__tgc__DataModelVersion())); }