Example #1
0
  // 处理对应的dbBatch
  private DbBatch getDbBatch(HttpPipeKey key) {
    String dataUrl = key.getUrl();
    Pipeline pipeline = configClientService.findPipeline(key.getIdentity().getPipelineId());
    DataRetriever dataRetriever =
        dataRetrieverFactory.createRetriever(
            pipeline.getParameters().getRetriever(), dataUrl, downloadDir);
    File archiveFile = null;
    try {
      dataRetriever.connect();
      dataRetriever.doRetrieve();
      archiveFile = dataRetriever.getDataAsFile();
    } catch (Exception e) {
      dataRetriever.abort();
      throw new PipeException("download_error", e);
    } finally {
      dataRetriever.disconnect();
    }

    // 处理下有加密的数据
    if (StringUtils.isNotEmpty(key.getKey()) && StringUtils.isNotEmpty(key.getCrc())) {
      decodeFile(archiveFile, key.getKey(), key.getCrc());
    }

    InputStream input = null;
    JSONReader reader = null;
    try {
      input = new BufferedInputStream(new FileInputStream(archiveFile));
      DbBatch dbBatch = new DbBatch();
      byte[] lengthBytes = new byte[4];
      input.read(lengthBytes);
      int length = ByteUtils.bytes2int(lengthBytes);
      BatchProto.RowBatch rowbatchProto =
          BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
      // 构造原始的model对象
      RowBatch rowBatch = new RowBatch();
      rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
      for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
        EventData eventData = new EventData();
        eventData.setPairId(rowDataProto.getPairId());
        eventData.setTableId(rowDataProto.getTableId());
        eventData.setTableName(rowDataProto.getTableName());
        eventData.setSchemaName(rowDataProto.getSchemaName());
        eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
        eventData.setExecuteTime(rowDataProto.getExecuteTime());
        // add by ljh at 2012-10-31
        if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
          eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
        }
        if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
          eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
        }
        // 处理主键
        List<EventColumn> keys = new ArrayList<EventColumn>();
        for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
          keys.add(buildColumn(columnProto));
        }
        eventData.setKeys(keys);
        // 处理old主键
        if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
          List<EventColumn> oldKeys = new ArrayList<EventColumn>();
          for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
            oldKeys.add(buildColumn(columnProto));
          }
          eventData.setOldKeys(oldKeys);
        }
        // 处理具体的column value
        List<EventColumn> columns = new ArrayList<EventColumn>();
        for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
          columns.add(buildColumn(columnProto));
        }
        eventData.setColumns(columns);

        eventData.setRemedy(rowDataProto.getRemedy());
        eventData.setSize(rowDataProto.getSize());
        // 添加到总记录
        rowBatch.merge(eventData);
      }
      dbBatch.setRowBatch(rowBatch);

      input.read(lengthBytes);
      length = ByteUtils.bytes2int(lengthBytes);
      BatchProto.FileBatch filebatchProto =
          BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
      // 构造原始的model对象
      FileBatch fileBatch = new FileBatch();
      fileBatch.setIdentity(build(filebatchProto.getIdentity()));
      for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
        FileData fileData = new FileData();
        fileData.setPairId(fileDataProto.getPairId());
        fileData.setTableId(fileDataProto.getTableId());
        fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
        fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
        fileData.setNameSpace(fileDataProto.getNamespace());
        fileData.setPath(fileDataProto.getPath());
        fileData.setSize(fileDataProto.getSize());
        // 添加到filebatch中
        fileBatch.getFiles().add(fileData);
      }
      dbBatch.setFileBatch(fileBatch);
      return dbBatch;
    } catch (IOException e) {
      throw new PipeException("deserial_error", e);
    } finally {
      IOUtils.closeQuietly(reader);
    }
  }
Example #2
0
  // ======================== help method ===================
  // 保存对应的dbBatch
  private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
    RowBatch rowBatch = dbBatch.getRowBatch();
    // 转化为proto对象
    BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
    rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
    // 处理具体的字段rowData
    for (EventData eventData : rowBatch.getDatas()) {
      BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
      rowDataBuilder.setPairId(eventData.getPairId());
      rowDataBuilder.setTableId(eventData.getTableId());
      if (eventData.getSchemaName() != null) {
        rowDataBuilder.setSchemaName(eventData.getSchemaName());
      }
      rowDataBuilder.setTableName(eventData.getTableName());
      rowDataBuilder.setEventType(eventData.getEventType().getValue());
      rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
      // add by ljh at 2012-10-31
      if (eventData.getSyncMode() != null) {
        rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
      }
      if (eventData.getSyncConsistency() != null) {
        rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
      }

      // 构造key column
      for (EventColumn keyColumn : eventData.getKeys()) {
        rowDataBuilder.addKeys(buildColumn(keyColumn));
      }
      // 构造old key column
      if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
        for (EventColumn keyColumn : eventData.getOldKeys()) {
          rowDataBuilder.addOldKeys(buildColumn(keyColumn));
        }
      }

      // 构造其他 column
      for (EventColumn column : eventData.getColumns()) {
        rowDataBuilder.addColumns(buildColumn(column));
      }

      rowDataBuilder.setRemedy(eventData.isRemedy());
      rowDataBuilder.setSize(eventData.getSize());
      rowBatchBuilder.addRows(rowDataBuilder.build()); // 添加一条rowData记录
    }

    // 处理下FileBatch
    FileBatch fileBatch = dbBatch.getFileBatch();
    BatchProto.FileBatch.Builder fileBatchBuilder = null;
    fileBatchBuilder = BatchProto.FileBatch.newBuilder();
    fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
    // 构造对应的proto对象
    for (FileData fileData : fileBatch.getFiles()) {
      BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
      fileDataBuilder.setPairId(fileData.getPairId());
      fileDataBuilder.setTableId(fileData.getTableId());
      if (fileData.getNameSpace() != null) {
        fileDataBuilder.setNamespace(fileData.getNameSpace());
      }
      if (fileData.getPath() != null) {
        fileDataBuilder.setPath(fileData.getPath());
      }
      fileDataBuilder.setEventType(fileData.getEventType().getValue());
      fileDataBuilder.setSize(fileData.getSize());
      fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());

      fileBatchBuilder.addFiles(fileDataBuilder.build()); // 添加一条fileData记录
    }
    // 处理构造对应的文件url
    String filename =
        buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
    // 写入数据
    File file = new File(htdocsDir, filename);
    OutputStream output = null;
    try {
      output = new BufferedOutputStream(new FileOutputStream(file));
      com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto =
          rowBatchBuilder.build();
      output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize())); // 输出大小
      rowBatchProto.writeTo(output); // 输出row batch

      com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto =
          fileBatchBuilder.build();
      output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize())); // 输出大小
      fileBatchProto.writeTo(output); // 输出file batch
      output.flush();
    } catch (IOException e) {
      throw new PipeException("write_byte_error", e);
    } finally {
      IOUtils.closeQuietly(output);
    }

    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.DB_BATCH);
    key.setIdentity(rowBatch.getIdentity());
    Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
    if (pipeline.getParameters().getUseFileEncrypt()) {
      // 加密处理
      EncryptedData encryptedData = encryptFile(file);
      key.setKey(encryptedData.getKey());
      key.setCrc(encryptedData.getCrc());
    }

    return key;
  }
Example #3
0
  /** 执行压缩 */
  private boolean doPack(
      final File targetArchiveFile,
      List<FileData> fileDatas,
      final ArchiveRetriverCallback<FileData> callback) {
    // 首先判断下对应的目标文件是否存在,如存在则执行删除
    if (true == targetArchiveFile.exists() && false == NioUtils.delete(targetArchiveFile, 3)) {
      throw new ArchiveException(
          String.format("[%s] exist and delete failed", targetArchiveFile.getAbsolutePath()));
    }

    boolean exist = false;
    ZipOutputStream zipOut = null;
    Set<String> entryNames = new HashSet<String>();
    BlockingQueue<Future<ArchiveEntry>> queue =
        new LinkedBlockingQueue<Future<ArchiveEntry>>(); // 下载成功的任务列表
    ExecutorCompletionService completionService = new ExecutorCompletionService(executor, queue);

    final File targetDir =
        new File(
            targetArchiveFile.getParentFile(),
            FilenameUtils.getBaseName(targetArchiveFile.getPath()));
    try {
      // 创建一个临时目录
      FileUtils.forceMkdir(targetDir);

      zipOut =
          new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(targetArchiveFile)));
      zipOut.setLevel(Deflater.BEST_SPEED);
      // 进行并发压缩处理
      for (final FileData fileData : fileDatas) {
        if (fileData.getEventType().isDelete()) {
          continue; // 忽略delete类型的数据打包,因为只需直接在目标进行删除
        }

        String namespace = fileData.getNameSpace();
        String path = fileData.getPath();
        boolean isLocal = StringUtils.isBlank(namespace);
        String entryName = null;
        if (true == isLocal) {
          entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
        } else {
          entryName = namespace + File.separator + path;
        }

        // 过滤一些重复的文件数据同步
        if (entryNames.contains(entryName) == false) {
          entryNames.add(entryName);
        } else {
          continue;
        }

        final String name = entryName;
        if (true == isLocal && !useLocalFileMutliThread) {
          // 采用串行处理,不走临时文件
          queue.add(new DummyFuture(new ArchiveEntry(name, callback.retrive(fileData))));
        } else {
          completionService.submit(
              new Callable<ArchiveEntry>() {

                public ArchiveEntry call() throws Exception {
                  // 处理下异常,可能失败
                  InputStream input = null;
                  OutputStream output = null;
                  try {
                    input = callback.retrive(fileData);

                    if (input instanceof LazyFileInputStream) {
                      input = ((LazyFileInputStream) input).getInputSteam(); // 获取原始的stream
                    }

                    if (input != null) {
                      File tmp = new File(targetDir, name);
                      NioUtils.create(tmp.getParentFile(), false, 3); // 尝试创建父路径
                      output = new FileOutputStream(tmp);
                      NioUtils.copy(input, output); // 拷贝到文件
                      return new ArchiveEntry(name, new File(targetDir, name));
                    } else {
                      return new ArchiveEntry(name);
                    }
                  } finally {
                    IOUtils.closeQuietly(input);
                    IOUtils.closeQuietly(output);
                  }
                }
              });
        }
      }

      for (int i = 0; i < entryNames.size(); i++) {
        // 读入流
        ArchiveEntry input = null;
        InputStream stream = null;
        try {
          input = queue.take().get();
          if (input == null) {
            continue;
          }

          stream = input.getStream();
          if (stream == null) {
            continue;
          }

          if (stream instanceof LazyFileInputStream) {
            stream = ((LazyFileInputStream) stream).getInputSteam(); // 获取原始的stream
          }

          exist = true;
          zipOut.putNextEntry(new ZipEntry(input.getName()));
          NioUtils.copy(stream, zipOut); // 输出到压缩流中
          zipOut.closeEntry();
        } finally {
          IOUtils.closeQuietly(stream);
        }
      }

      if (exist) {
        zipOut.finish();
      }
    } catch (Exception e) {
      throw new ArchiveException(e);
    } finally {
      IOUtils.closeQuietly(zipOut);
      try {
        FileUtils.deleteDirectory(targetDir); // 删除临时目录
      } catch (IOException e) {
        // ignore
      }
    }

    return exist;
  }