// ======================== help method =================== // 保存对应的dbBatch private HttpPipeKey saveDbBatch(DbBatch dbBatch) { RowBatch rowBatch = dbBatch.getRowBatch(); // 转化为proto对象 BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder(); rowBatchBuilder.setIdentity(build(rowBatch.getIdentity())); // 处理具体的字段rowData for (EventData eventData : rowBatch.getDatas()) { BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder(); rowDataBuilder.setPairId(eventData.getPairId()); rowDataBuilder.setTableId(eventData.getTableId()); if (eventData.getSchemaName() != null) { rowDataBuilder.setSchemaName(eventData.getSchemaName()); } rowDataBuilder.setTableName(eventData.getTableName()); rowDataBuilder.setEventType(eventData.getEventType().getValue()); rowDataBuilder.setExecuteTime(eventData.getExecuteTime()); // add by ljh at 2012-10-31 if (eventData.getSyncMode() != null) { rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue()); } if (eventData.getSyncConsistency() != null) { rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue()); } // 构造key column for (EventColumn keyColumn : eventData.getKeys()) { rowDataBuilder.addKeys(buildColumn(keyColumn)); } // 构造old key column if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) { for (EventColumn keyColumn : eventData.getOldKeys()) { rowDataBuilder.addOldKeys(buildColumn(keyColumn)); } } // 构造其他 column for (EventColumn column : eventData.getColumns()) { rowDataBuilder.addColumns(buildColumn(column)); } rowDataBuilder.setRemedy(eventData.isRemedy()); rowDataBuilder.setSize(eventData.getSize()); rowBatchBuilder.addRows(rowDataBuilder.build()); // 添加一条rowData记录 } // 处理下FileBatch FileBatch fileBatch = dbBatch.getFileBatch(); BatchProto.FileBatch.Builder fileBatchBuilder = null; fileBatchBuilder = BatchProto.FileBatch.newBuilder(); fileBatchBuilder.setIdentity(build(fileBatch.getIdentity())); // 构造对应的proto对象 for (FileData fileData : fileBatch.getFiles()) { BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder(); fileDataBuilder.setPairId(fileData.getPairId()); fileDataBuilder.setTableId(fileData.getTableId()); if (fileData.getNameSpace() != null) { fileDataBuilder.setNamespace(fileData.getNameSpace()); } if (fileData.getPath() != null) { fileDataBuilder.setPath(fileData.getPath()); } fileDataBuilder.setEventType(fileData.getEventType().getValue()); fileDataBuilder.setSize(fileData.getSize()); fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime()); fileBatchBuilder.addFiles(fileDataBuilder.build()); // 添加一条fileData记录 } // 处理构造对应的文件url String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass())); // 写入数据 File file = new File(htdocsDir, filename); OutputStream output = null; try { output = new BufferedOutputStream(new FileOutputStream(file)); com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build(); output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize())); // 输出大小 rowBatchProto.writeTo(output); // 输出row batch com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build(); output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize())); // 输出大小 fileBatchProto.writeTo(output); // 输出file batch output.flush(); } catch (IOException e) { throw new PipeException("write_byte_error", e); } finally { IOUtils.closeQuietly(output); } HttpPipeKey key = new HttpPipeKey(); key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename)); key.setDataType(PipeDataType.DB_BATCH); key.setIdentity(rowBatch.getIdentity()); Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId()); if (pipeline.getParameters().getUseFileEncrypt()) { // 加密处理 EncryptedData encryptedData = encryptFile(file); key.setKey(encryptedData.getKey()); key.setCrc(encryptedData.getCrc()); } return key; }