// ======================== help method =================== // 保存对应的dbBatch private HttpPipeKey saveDbBatch(DbBatch dbBatch) { RowBatch rowBatch = dbBatch.getRowBatch(); // 转化为proto对象 BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder(); rowBatchBuilder.setIdentity(build(rowBatch.getIdentity())); // 处理具体的字段rowData for (EventData eventData : rowBatch.getDatas()) { BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder(); rowDataBuilder.setPairId(eventData.getPairId()); rowDataBuilder.setTableId(eventData.getTableId()); if (eventData.getSchemaName() != null) { rowDataBuilder.setSchemaName(eventData.getSchemaName()); } rowDataBuilder.setTableName(eventData.getTableName()); rowDataBuilder.setEventType(eventData.getEventType().getValue()); rowDataBuilder.setExecuteTime(eventData.getExecuteTime()); // add by ljh at 2012-10-31 if (eventData.getSyncMode() != null) { rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue()); } if (eventData.getSyncConsistency() != null) { rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue()); } // 构造key column for (EventColumn keyColumn : eventData.getKeys()) { rowDataBuilder.addKeys(buildColumn(keyColumn)); } // 构造old key column if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) { for (EventColumn keyColumn : eventData.getOldKeys()) { rowDataBuilder.addOldKeys(buildColumn(keyColumn)); } } // 构造其他 column for (EventColumn column : eventData.getColumns()) { rowDataBuilder.addColumns(buildColumn(column)); } rowDataBuilder.setRemedy(eventData.isRemedy()); rowDataBuilder.setSize(eventData.getSize()); rowBatchBuilder.addRows(rowDataBuilder.build()); // 添加一条rowData记录 } // 处理下FileBatch FileBatch fileBatch = dbBatch.getFileBatch(); BatchProto.FileBatch.Builder fileBatchBuilder = null; fileBatchBuilder = BatchProto.FileBatch.newBuilder(); fileBatchBuilder.setIdentity(build(fileBatch.getIdentity())); // 构造对应的proto对象 for (FileData fileData : fileBatch.getFiles()) { BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder(); fileDataBuilder.setPairId(fileData.getPairId()); fileDataBuilder.setTableId(fileData.getTableId()); if (fileData.getNameSpace() != null) { fileDataBuilder.setNamespace(fileData.getNameSpace()); } if (fileData.getPath() != null) { fileDataBuilder.setPath(fileData.getPath()); } fileDataBuilder.setEventType(fileData.getEventType().getValue()); fileDataBuilder.setSize(fileData.getSize()); fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime()); fileBatchBuilder.addFiles(fileDataBuilder.build()); // 添加一条fileData记录 } // 处理构造对应的文件url String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass())); // 写入数据 File file = new File(htdocsDir, filename); OutputStream output = null; try { output = new BufferedOutputStream(new FileOutputStream(file)); com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build(); output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize())); // 输出大小 rowBatchProto.writeTo(output); // 输出row batch com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build(); output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize())); // 输出大小 fileBatchProto.writeTo(output); // 输出file batch output.flush(); } catch (IOException e) { throw new PipeException("write_byte_error", e); } finally { IOUtils.closeQuietly(output); } HttpPipeKey key = new HttpPipeKey(); key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename)); key.setDataType(PipeDataType.DB_BATCH); key.setIdentity(rowBatch.getIdentity()); Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId()); if (pipeline.getParameters().getUseFileEncrypt()) { // 加密处理 EncryptedData encryptedData = encryptFile(file); key.setKey(encryptedData.getKey()); key.setCrc(encryptedData.getCrc()); } return key; }
/** 执行压缩 */ private boolean doPack( final File targetArchiveFile, List<FileData> fileDatas, final ArchiveRetriverCallback<FileData> callback) { // 首先判断下对应的目标文件是否存在,如存在则执行删除 if (true == targetArchiveFile.exists() && false == NioUtils.delete(targetArchiveFile, 3)) { throw new ArchiveException( String.format("[%s] exist and delete failed", targetArchiveFile.getAbsolutePath())); } boolean exist = false; ZipOutputStream zipOut = null; Set<String> entryNames = new HashSet<String>(); BlockingQueue<Future<ArchiveEntry>> queue = new LinkedBlockingQueue<Future<ArchiveEntry>>(); // 下载成功的任务列表 ExecutorCompletionService completionService = new ExecutorCompletionService(executor, queue); final File targetDir = new File( targetArchiveFile.getParentFile(), FilenameUtils.getBaseName(targetArchiveFile.getPath())); try { // 创建一个临时目录 FileUtils.forceMkdir(targetDir); zipOut = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(targetArchiveFile))); zipOut.setLevel(Deflater.BEST_SPEED); // 进行并发压缩处理 for (final FileData fileData : fileDatas) { if (fileData.getEventType().isDelete()) { continue; // 忽略delete类型的数据打包,因为只需直接在目标进行删除 } String namespace = fileData.getNameSpace(); String path = fileData.getPath(); boolean isLocal = StringUtils.isBlank(namespace); String entryName = null; if (true == isLocal) { entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path); } else { entryName = namespace + File.separator + path; } // 过滤一些重复的文件数据同步 if (entryNames.contains(entryName) == false) { entryNames.add(entryName); } else { continue; } final String name = entryName; if (true == isLocal && !useLocalFileMutliThread) { // 采用串行处理,不走临时文件 queue.add(new DummyFuture(new ArchiveEntry(name, callback.retrive(fileData)))); } else { completionService.submit( new Callable<ArchiveEntry>() { public ArchiveEntry call() throws Exception { // 处理下异常,可能失败 InputStream input = null; OutputStream output = null; try { input = callback.retrive(fileData); if (input instanceof LazyFileInputStream) { input = ((LazyFileInputStream) input).getInputSteam(); // 获取原始的stream } if (input != null) { File tmp = new File(targetDir, name); NioUtils.create(tmp.getParentFile(), false, 3); // 尝试创建父路径 output = new FileOutputStream(tmp); NioUtils.copy(input, output); // 拷贝到文件 return new ArchiveEntry(name, new File(targetDir, name)); } else { return new ArchiveEntry(name); } } finally { IOUtils.closeQuietly(input); IOUtils.closeQuietly(output); } } }); } } for (int i = 0; i < entryNames.size(); i++) { // 读入流 ArchiveEntry input = null; InputStream stream = null; try { input = queue.take().get(); if (input == null) { continue; } stream = input.getStream(); if (stream == null) { continue; } if (stream instanceof LazyFileInputStream) { stream = ((LazyFileInputStream) stream).getInputSteam(); // 获取原始的stream } exist = true; zipOut.putNextEntry(new ZipEntry(input.getName())); NioUtils.copy(stream, zipOut); // 输出到压缩流中 zipOut.closeEntry(); } finally { IOUtils.closeQuietly(stream); } } if (exist) { zipOut.finish(); } } catch (Exception e) { throw new ArchiveException(e); } finally { IOUtils.closeQuietly(zipOut); try { FileUtils.deleteDirectory(targetDir); // 删除临时目录 } catch (IOException e) { // ignore } } return exist; }