// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
RowBatch rowBatch = dbBatch.getRowBatch();
// 转化为proto对象
BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
// 处理具体的字段rowData
for (EventData eventData : rowBatch.getDatas()) {
BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
rowDataBuilder.setPairId(eventData.getPairId());
rowDataBuilder.setTableId(eventData.getTableId());
if (eventData.getSchemaName() != null) {
rowDataBuilder.setSchemaName(eventData.getSchemaName());
}
rowDataBuilder.setTableName(eventData.getTableName());
rowDataBuilder.setEventType(eventData.getEventType().getValue());
rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
// add by ljh at 2012-10-31
if (eventData.getSyncMode() != null) {
rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
}
if (eventData.getSyncConsistency() != null) {
rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
}
// 构造key column
for (EventColumn keyColumn : eventData.getKeys()) {
rowDataBuilder.addKeys(buildColumn(keyColumn));
}
// 构造old key column
if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
for (EventColumn keyColumn : eventData.getOldKeys()) {
rowDataBuilder.addOldKeys(buildColumn(keyColumn));
}
}
// 构造其他 column
for (EventColumn column : eventData.getColumns()) {
rowDataBuilder.addColumns(buildColumn(column));
}
rowDataBuilder.setRemedy(eventData.isRemedy());
rowDataBuilder.setSize(eventData.getSize());
rowBatchBuilder.addRows(rowDataBuilder.build()); // 添加一条rowData记录
}
// 处理下FileBatch
FileBatch fileBatch = dbBatch.getFileBatch();
BatchProto.FileBatch.Builder fileBatchBuilder = null;
fileBatchBuilder = BatchProto.FileBatch.newBuilder();
fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
// 构造对应的proto对象
for (FileData fileData : fileBatch.getFiles()) {
BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
fileDataBuilder.setPairId(fileData.getPairId());
fileDataBuilder.setTableId(fileData.getTableId());
if (fileData.getNameSpace() != null) {
fileDataBuilder.setNamespace(fileData.getNameSpace());
}
if (fileData.getPath() != null) {
fileDataBuilder.setPath(fileData.getPath());
}
fileDataBuilder.setEventType(fileData.getEventType().getValue());
fileDataBuilder.setSize(fileData.getSize());
fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
fileBatchBuilder.addFiles(fileDataBuilder.build()); // 添加一条fileData记录
}
// 处理构造对应的文件url
String filename =
buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
// 写入数据
File file = new File(htdocsDir, filename);
OutputStream output = null;
try {
output = new BufferedOutputStream(new FileOutputStream(file));
com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto =
rowBatchBuilder.build();
output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize())); // 输出大小
rowBatchProto.writeTo(output); // 输出row batch
com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto =
fileBatchBuilder.build();
output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize())); // 输出大小
fileBatchProto.writeTo(output); // 输出file batch
output.flush();
} catch (IOException e) {
throw new PipeException("write_byte_error", e);
} finally {
IOUtils.closeQuietly(output);
}
HttpPipeKey key = new HttpPipeKey();
key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
key.setDataType(PipeDataType.DB_BATCH);
key.setIdentity(rowBatch.getIdentity());
Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
if (pipeline.getParameters().getUseFileEncrypt()) {
// 加密处理
EncryptedData encryptedData = encryptFile(file);
key.setKey(encryptedData.getKey());
key.setCrc(encryptedData.getCrc());
}
return key;
}
