/**
* 標準入力を読み込んでDFSにファイルを書き出す。 {@link FileList}形式で受け取ったTSV形式のファイルをModelオブジェクトに変換して、テンポラリ入力データとして配置する。
* 出力先はプロトコルの形式によって異なる。 利用可能なプロトコルは以下のとおり。
*
* <ul>
* <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#CONTENT}
* <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#CREATE_CACHE}
* <li>{@link com.asakusafw.bulkloader.transfer.FileProtocol.Kind#UPDATE_CACHE}
* </ul>
*
* @param bean パラメータを保持するBean
* @param user OSのユーザー名
* @return 出力結果（true：正常終了、false：異常終了）
*/
public boolean importFile(ImportBean bean, String user) {
// 標準入力を取得
FileList.Reader reader;
try {
reader = FileList.createReader(getInputStream());
} catch (IOException e) {
LOG.error(e, "TG-EXTRACTOR-02001", "標準入力からFileListの取得に失敗");
return false;
}
try {
// FileListの終端まで繰り返す
List<Future<?>> running = new ArrayList<>();
while (reader.next()) {
FileProtocol protocol = reader.getCurrentProtocol();
try (InputStream content = reader.openContent()) {
switch (protocol.getKind()) {
case CONTENT:
importContent(protocol, content, bean, user);
break;
case CREATE_CACHE:
case UPDATE_CACHE:
long recordCount = putCachePatch(protocol, content, bean, user);
Callable<?> builder = createCacheBuilder(protocol, bean, user, recordCount);
if (builder != null) {
LOG.debugMessage(
"Submitting cache builder: {0} {1}",
protocol.getKind(), protocol.getInfo().getTableName());
running.add(executor.submit(builder));
}
break;
default:
throw new AssertionError(protocol.getKind());
}
}
}
waitForCompleteTasks(bean, running);
// 正常終了
return true;
} catch (BulkLoaderSystemException e) {
LOG.log(e);
} catch (IOException e) {
// FileListの展開に失敗
LOG.error(e, "TG-EXTRACTOR-02001", "標準入力からFileListの取得に失敗");
} finally {
try {
reader.close();
} catch (IOException e) {
// ここで例外が発生した場合は握りつぶす
e.printStackTrace();
}
}
return false;
}
private long putCachePatch(
FileProtocol protocol, InputStream content, ImportBean bean, String user)
throws BulkLoaderSystemException {
assert protocol != null;
assert content != null;
assert bean != null;
assert user != null;
assert protocol.getKind() == FileProtocol.Kind.CREATE_CACHE
|| protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;
CacheInfo info = protocol.getInfo();
assert info != null;
ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
if (targetTableBean == null) {
// 対応するテーブルの定義がDSL存在しない場合異常終了する。
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-02001",
MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}", info.getTableName()));
}
URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
try (CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath)) {
LOG.info(
"TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
storage.putPatchCacheInfo(info);
LOG.info(
"TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());
Class<?> targetTableModel = targetTableBean.getImportTargetType();
Path targetUri = storage.getPatchContents("0");
LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
long recordCount = write(targetTableModel, targetUri.toUri(), content);
LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
LOG.info(
"TG-PROFILE-01002",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
info.getTableName(),
recordCount);
return recordCount;
} catch (IOException e) {
throw new BulkLoaderSystemException(
e, getClass(), "TG-EXTRACTOR-11005", info.getId(), info.getTableName(), dfsFilePath);
}
}
/**
* Collectorの処理を実行する。
*
* @param args コマンドライン引数
* @return 処理全体の終了コード
* @see Constants#EXIT_CODE_SUCCESS
* @see Constants#EXIT_CODE_WARNING
* @see Constants#EXIT_CODE_ERROR
*/
protected int execute(String[] args) {
if (args.length != 5) {
System.err.println("Collectorに指定する引数の数が不正です。 引数の数：" + args.length);
return Constants.EXIT_CODE_ERROR;
}
String targetName = args[0];
String batchId = args[1];
String jobflowId = args[2];
String executionId = args[3];
String user = args[4];
try {
// 初期処理
if (!BulkLoaderInitializer.initHadoopCluster(jobflowId, executionId, PROPERTIES)) {
LOG.error(
"TG-COLLECTOR-01003", new Date(), targetName, batchId, jobflowId, executionId, user);
return Constants.EXIT_CODE_ERROR;
}
// 開始ログ出力
LOG.info("TG-COLLECTOR-01001", new Date(), targetName, batchId, jobflowId, executionId, user);
// パラメータオブジェクトを作成
ExporterBean bean = createBean(targetName, batchId, jobflowId, executionId);
if (bean == null) {
// パラメータのチェックでエラー
LOG.error(
"TG-COLLECTOR-01006", new Date(), targetName, batchId, jobflowId, executionId, user);
return Constants.EXIT_CODE_ERROR;
}
// ExportファイルをDBサーバに送信
LOG.info("TG-COLLECTOR-01008", targetName, batchId, jobflowId, executionId, user);
ExportFileSend fileSend = createExportFileSend();
if (!fileSend.sendExportFile(bean, user)) {
// Exportファイルの送信に失敗
LOG.error(
"TG-COLLECTOR-01007", new Date(), targetName, batchId, jobflowId, executionId, user);
return Constants.EXIT_CODE_ERROR;
} else {
LOG.info("TG-COLLECTOR-01009", targetName, batchId, jobflowId, executionId, user);
}
// 正常終了
LOG.info("TG-COLLECTOR-01002", new Date(), targetName, batchId, jobflowId, executionId, user);
return Constants.EXIT_CODE_SUCCESS;
} catch (Exception e) {
try {
LOG.error(
e, "TG-COLLECTOR-01004", new Date(), targetName, batchId, jobflowId, executionId, user);
return Constants.EXIT_CODE_ERROR;
} catch (Exception e1) {
System.err.print("Collectorで不明なエラーが発生しました。");
e1.printStackTrace();
return Constants.EXIT_CODE_ERROR;
}
}
}
/**
* Creates a cache builder for the specified cache (of candidate).
*
* @param subcommand subcommand name
* @param bean current importer script
* @param location cache location
* @param info cache information
* @return the future object of the execution, or {@code null} if nothing to do
* @throws IOException if failed to start execution
*/
protected Callable<?> createCacheBuilder(
final String subcommand, ImportBean bean, final URI location, final CacheInfo info)
throws IOException {
assert subcommand != null;
assert bean != null;
assert location != null;
assert info != null;
List<String> command = new ArrayList<>();
command.add(cacheBuildCommand);
command.add(subcommand);
command.add(bean.getBatchId());
command.add(bean.getJobflowId());
command.add(bean.getExecutionId());
command.add(location.toString());
command.add(info.getModelClassName());
command.add(info.getTableName());
LOG.info(
"TG-EXTRACTOR-12001",
subcommand,
info.getId(),
info.getTableName(),
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
command);
final ProcessBuilder builder = new ProcessBuilder(command);
builder.directory(new File(System.getProperty("user.home", ".")));
return new Callable<Void>() {
@Override
public Void call() throws Exception {
LOG.info("TG-EXTRACTOR-12003", subcommand, info.getId(), info.getTableName());
Process process =
builder.redirectOutput(Redirect.INHERIT).redirectError(Redirect.INHERIT).start();
try {
int exitCode = process.waitFor();
if (exitCode != 0) {
throw new IOException(
MessageFormat.format("Cache builder returns unexpected exit code: {0}", exitCode));
}
LOG.info("TG-EXTRACTOR-12004", subcommand, info.getId(), info.getTableName());
} catch (Exception e) {
throw new BulkLoaderSystemException(
e,
DfsFileImport.class,
"TG-EXTRACTOR-12005",
subcommand,
info.getId(),
info.getTableName());
} finally {
process.destroy();
}
return null;
}
};
}
/** Creates a new instance. */
public DfsFileImport() {
File cmd = ConfigurationLoader.getLocalScriptPath(Constants.PATH_LOCAL_CACHE_BUILD);
this.cacheBuildCommand = cmd.getAbsolutePath();
int parallel =
Integer.parseInt(
ConfigurationLoader.getProperty(Constants.PROP_KEY_CACHE_BUILDER_PARALLEL));
LOG.debugMessage("Building a cache builder with {0} threads", parallel);
this.executor = Executors.newFixedThreadPool(parallel);
}
/**
* 圧縮の種類を取得する。
*
* @param strCompType CompressionTypeの文字列
* @return CompressionType
*/
protected CompressionType getCompType(String strCompType) {
CompressionType compType = null;
try {
compType = CompressionType.valueOf(strCompType);
} catch (Exception e) {
compType = CompressionType.NONE;
LOG.warn("TG-EXTRACTOR-02004", strCompType);
}
return compType;
}
/**
* loadJDBCPropのテストケース 異常系：DBMSの接続情報を記述したプロパティファイルの読み込みに失敗するケース
*
* @throws Exception
*/
@Test
public void loadJDBCProp02() throws Exception {
ConfigurationLoader.init(properties_db, true, false);
//
// LogInitializer.execute(ConfigurationLoader.getProperty(Constants.PROP_KEY_LOG_CONF_PATH));
try {
ConfigurationLoader.loadJDBCProp("targetErr");
fail();
} catch (BulkLoaderSystemException e) {
LOG.info(e.getCause(), e.getMessageId(), e.getMessageArgs());
}
}
private void importContent(
FileProtocol protocol, InputStream content, ImportBean bean, String user)
throws BulkLoaderSystemException {
assert protocol != null;
assert content != null;
assert bean != null;
assert user != null;
String tableName = FileNameUtil.getImportTableName(protocol.getLocation());
ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName);
if (targetTableBean == null) {
// 対応するテーブルの定義がDSL存在しない場合異常終了する。
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-02001",
MessageFormat.format("エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}", tableName));
}
URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath());
Class<?> targetTableModel = targetTableBean.getImportTargetType();
LOG.info("TG-EXTRACTOR-02002", tableName, dfsFilePath.toString(), targetTableModel.toString());
// ファイルをジョブ入力データ領域に書き出す
long recordCount = write(targetTableModel, dfsFilePath, content);
LOG.info("TG-EXTRACTOR-02003", tableName, dfsFilePath.toString(), targetTableModel.toString());
LOG.info(
"TG-PROFILE-01002",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId(),
tableName,
recordCount);
}
/**
* loadJDBCPropのテストケース 異常系：DBMSのDB名が設定されていないケース
*
* @throws Exception
*/
@Test
public void loadJDBCProp07() throws Exception {
// ConfigurationLoader.init(propertys_db, true, false);
//
// LogInitializer.execute(ConfigurationLoader.getProperty(Constants.PROP_KEY_LOG_CONF_PATH));
ConfigurationLoader.init(properties_db, false, false);
Properties p = ConfigurationLoader.getProperty();
p.setProperty("jdbc.driver", "com.mysql.jdbc.Driver");
p.setProperty("jdbc.url", "jdbc:mysql://localhost/asakusa");
p.setProperty("jdbc.user", "asakusa");
p.setProperty("jdbc.password", "asakusa");
p.setProperty("db.parameter", PATH_DB_PARAMETER);
ConfigurationLoader.setProperty(p);
try {
ConfigurationLoader.loadJDBCProp("target2");
} catch (BulkLoaderSystemException e) {
LOG.info(e.getCause(), e.getMessageId(), e.getMessageArgs());
}
}
private void waitForCompleteTasks(ImportBean bean, List<Future<?>> running)
throws BulkLoaderSystemException {
assert bean != null;
assert running != null;
if (running.isEmpty()) {
return;
}
LOG.info(
"TG-EXTRACTOR-12006",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId());
boolean sawError = false;
LinkedList<Future<?>> rest = new LinkedList<>(running);
while (rest.isEmpty() == false) {
Future<?> future = rest.removeFirst();
try {
future.get(1, TimeUnit.SECONDS);
} catch (TimeoutException e) {
// continue...
rest.addLast(future);
} catch (InterruptedException e) {
cancel(rest);
throw new BulkLoaderSystemException(
e,
getClass(),
"TG-EXTRACTOR-12007",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId());
} catch (ExecutionException e) {
cancel(rest);
Throwable cause = e.getCause();
if (cause instanceof RuntimeException) {
throw (RuntimeException) cause;
} else if (cause instanceof Error) {
throw (Error) cause;
} else if (cause instanceof BulkLoaderSystemException) {
LOG.log((BulkLoaderSystemException) cause);
sawError = true;
} else {
LOG.error(
e,
"TG-EXTRACTOR-12008",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId());
sawError = true;
}
}
}
if (sawError) {
throw new BulkLoaderSystemException(
getClass(),
"TG-EXTRACTOR-12008",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId());
} else {
LOG.info(
"TG-EXTRACTOR-12009",
bean.getTargetName(),
bean.getBatchId(),
bean.getJobflowId(),
bean.getExecutionId());
}
}