// 处理对应的dbBatch
private DbBatch getDbBatch(HttpPipeKey key) {
String dataUrl = key.getUrl();
Pipeline pipeline = configClientService.findPipeline(key.getIdentity().getPipelineId());
DataRetriever dataRetriever =
dataRetrieverFactory.createRetriever(
pipeline.getParameters().getRetriever(), dataUrl, downloadDir);
File archiveFile = null;
try {
dataRetriever.connect();
dataRetriever.doRetrieve();
archiveFile = dataRetriever.getDataAsFile();
} catch (Exception e) {
dataRetriever.abort();
throw new PipeException("download_error", e);
} finally {
dataRetriever.disconnect();
}
// 处理下有加密的数据
if (StringUtils.isNotEmpty(key.getKey()) && StringUtils.isNotEmpty(key.getCrc())) {
decodeFile(archiveFile, key.getKey(), key.getCrc());
}
InputStream input = null;
JSONReader reader = null;
try {
input = new BufferedInputStream(new FileInputStream(archiveFile));
DbBatch dbBatch = new DbBatch();
byte[] lengthBytes = new byte[4];
input.read(lengthBytes);
int length = ByteUtils.bytes2int(lengthBytes);
BatchProto.RowBatch rowbatchProto =
BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
// 构造原始的model对象
RowBatch rowBatch = new RowBatch();
rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
EventData eventData = new EventData();
eventData.setPairId(rowDataProto.getPairId());
eventData.setTableId(rowDataProto.getTableId());
eventData.setTableName(rowDataProto.getTableName());
eventData.setSchemaName(rowDataProto.getSchemaName());
eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
eventData.setExecuteTime(rowDataProto.getExecuteTime());
// add by ljh at 2012-10-31
if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
}
if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
}
// 处理主键
List<EventColumn> keys = new ArrayList<EventColumn>();
for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
keys.add(buildColumn(columnProto));
}
eventData.setKeys(keys);
// 处理old主键
if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
List<EventColumn> oldKeys = new ArrayList<EventColumn>();
for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
oldKeys.add(buildColumn(columnProto));
}
eventData.setOldKeys(oldKeys);
}
// 处理具体的column value
List<EventColumn> columns = new ArrayList<EventColumn>();
for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
columns.add(buildColumn(columnProto));
}
eventData.setColumns(columns);
eventData.setRemedy(rowDataProto.getRemedy());
eventData.setSize(rowDataProto.getSize());
// 添加到总记录
rowBatch.merge(eventData);
}
dbBatch.setRowBatch(rowBatch);
input.read(lengthBytes);
length = ByteUtils.bytes2int(lengthBytes);
BatchProto.FileBatch filebatchProto =
BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
// 构造原始的model对象
FileBatch fileBatch = new FileBatch();
fileBatch.setIdentity(build(filebatchProto.getIdentity()));
for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
FileData fileData = new FileData();
fileData.setPairId(fileDataProto.getPairId());
fileData.setTableId(fileDataProto.getTableId());
fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
fileData.setNameSpace(fileDataProto.getNamespace());
fileData.setPath(fileDataProto.getPath());
fileData.setSize(fileDataProto.getSize());
// 添加到filebatch中
fileBatch.getFiles().add(fileData);
}
dbBatch.setFileBatch(fileBatch);
return dbBatch;
} catch (IOException e) {
throw new PipeException("deserial_error", e);
} finally {
IOUtils.closeQuietly(reader);
}
}
// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
RowBatch rowBatch = dbBatch.getRowBatch();
// 转化为proto对象
BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
// 处理具体的字段rowData
for (EventData eventData : rowBatch.getDatas()) {
BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
rowDataBuilder.setPairId(eventData.getPairId());
rowDataBuilder.setTableId(eventData.getTableId());
if (eventData.getSchemaName() != null) {
rowDataBuilder.setSchemaName(eventData.getSchemaName());
}
rowDataBuilder.setTableName(eventData.getTableName());
rowDataBuilder.setEventType(eventData.getEventType().getValue());
rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
// add by ljh at 2012-10-31
if (eventData.getSyncMode() != null) {
rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
}
if (eventData.getSyncConsistency() != null) {
rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
}
// 构造key column
for (EventColumn keyColumn : eventData.getKeys()) {
rowDataBuilder.addKeys(buildColumn(keyColumn));
}
// 构造old key column
if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
for (EventColumn keyColumn : eventData.getOldKeys()) {
rowDataBuilder.addOldKeys(buildColumn(keyColumn));
}
}
// 构造其他 column
for (EventColumn column : eventData.getColumns()) {
rowDataBuilder.addColumns(buildColumn(column));
}
rowDataBuilder.setRemedy(eventData.isRemedy());
rowDataBuilder.setSize(eventData.getSize());
rowBatchBuilder.addRows(rowDataBuilder.build()); // 添加一条rowData记录
}
// 处理下FileBatch
FileBatch fileBatch = dbBatch.getFileBatch();
BatchProto.FileBatch.Builder fileBatchBuilder = null;
fileBatchBuilder = BatchProto.FileBatch.newBuilder();
fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
// 构造对应的proto对象
for (FileData fileData : fileBatch.getFiles()) {
BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
fileDataBuilder.setPairId(fileData.getPairId());
fileDataBuilder.setTableId(fileData.getTableId());
if (fileData.getNameSpace() != null) {
fileDataBuilder.setNamespace(fileData.getNameSpace());
}
if (fileData.getPath() != null) {
fileDataBuilder.setPath(fileData.getPath());
}
fileDataBuilder.setEventType(fileData.getEventType().getValue());
fileDataBuilder.setSize(fileData.getSize());
fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
fileBatchBuilder.addFiles(fileDataBuilder.build()); // 添加一条fileData记录
}
// 处理构造对应的文件url
String filename =
buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
// 写入数据
File file = new File(htdocsDir, filename);
OutputStream output = null;
try {
output = new BufferedOutputStream(new FileOutputStream(file));
com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto =
rowBatchBuilder.build();
output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize())); // 输出大小
rowBatchProto.writeTo(output); // 输出row batch
com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto =
fileBatchBuilder.build();
output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize())); // 输出大小
fileBatchProto.writeTo(output); // 输出file batch
output.flush();
} catch (IOException e) {
throw new PipeException("write_byte_error", e);
} finally {
IOUtils.closeQuietly(output);
}
HttpPipeKey key = new HttpPipeKey();
key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
key.setDataType(PipeDataType.DB_BATCH);
key.setIdentity(rowBatch.getIdentity());
Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
if (pipeline.getParameters().getUseFileEncrypt()) {
// 加密处理
EncryptedData encryptedData = encryptFile(file);
key.setKey(encryptedData.getKey());
key.setCrc(encryptedData.getCrc());
}
return key;
}
/** 执行压缩 */
private boolean doPack(
final File targetArchiveFile,
List<FileData> fileDatas,
final ArchiveRetriverCallback<FileData> callback) {
// 首先判断下对应的目标文件是否存在，如存在则执行删除
if (true == targetArchiveFile.exists() && false == NioUtils.delete(targetArchiveFile, 3)) {
throw new ArchiveException(
String.format("[%s] exist and delete failed", targetArchiveFile.getAbsolutePath()));
}
boolean exist = false;
ZipOutputStream zipOut = null;
Set<String> entryNames = new HashSet<String>();
BlockingQueue<Future<ArchiveEntry>> queue =
new LinkedBlockingQueue<Future<ArchiveEntry>>(); // 下载成功的任务列表
ExecutorCompletionService completionService = new ExecutorCompletionService(executor, queue);
final File targetDir =
new File(
targetArchiveFile.getParentFile(),
FilenameUtils.getBaseName(targetArchiveFile.getPath()));
try {
// 创建一个临时目录
FileUtils.forceMkdir(targetDir);
zipOut =
new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(targetArchiveFile)));
zipOut.setLevel(Deflater.BEST_SPEED);
// 进行并发压缩处理
for (final FileData fileData : fileDatas) {
if (fileData.getEventType().isDelete()) {
continue; // 忽略delete类型的数据打包，因为只需直接在目标进行删除
}
String namespace = fileData.getNameSpace();
String path = fileData.getPath();
boolean isLocal = StringUtils.isBlank(namespace);
String entryName = null;
if (true == isLocal) {
entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
} else {
entryName = namespace + File.separator + path;
}
// 过滤一些重复的文件数据同步
if (entryNames.contains(entryName) == false) {
entryNames.add(entryName);
} else {
continue;
}
final String name = entryName;
if (true == isLocal && !useLocalFileMutliThread) {
// 采用串行处理，不走临时文件
queue.add(new DummyFuture(new ArchiveEntry(name, callback.retrive(fileData))));
} else {
completionService.submit(
new Callable<ArchiveEntry>() {
public ArchiveEntry call() throws Exception {
// 处理下异常，可能失败
InputStream input = null;
OutputStream output = null;
try {
input = callback.retrive(fileData);
if (input instanceof LazyFileInputStream) {
input = ((LazyFileInputStream) input).getInputSteam(); // 获取原始的stream
}
if (input != null) {
File tmp = new File(targetDir, name);
NioUtils.create(tmp.getParentFile(), false, 3); // 尝试创建父路径
output = new FileOutputStream(tmp);
NioUtils.copy(input, output); // 拷贝到文件
return new ArchiveEntry(name, new File(targetDir, name));
} else {
return new ArchiveEntry(name);
}
} finally {
IOUtils.closeQuietly(input);
IOUtils.closeQuietly(output);
}
}
});
}
}
for (int i = 0; i < entryNames.size(); i++) {
// 读入流
ArchiveEntry input = null;
InputStream stream = null;
try {
input = queue.take().get();
if (input == null) {
continue;
}
stream = input.getStream();
if (stream == null) {
continue;
}
if (stream instanceof LazyFileInputStream) {
stream = ((LazyFileInputStream) stream).getInputSteam(); // 获取原始的stream
}
exist = true;
zipOut.putNextEntry(new ZipEntry(input.getName()));
NioUtils.copy(stream, zipOut); // 输出到压缩流中
zipOut.closeEntry();
} finally {
IOUtils.closeQuietly(stream);
}
}
if (exist) {
zipOut.finish();
}
} catch (Exception e) {
throw new ArchiveException(e);
} finally {
IOUtils.closeQuietly(zipOut);
try {
FileUtils.deleteDirectory(targetDir); // 删除临时目录
} catch (IOException e) {
// ignore
}
}
return exist;
}