コード例 #1
0
  /**
   * @param pConfig The physical source config for which the event producer is configured.
   * @param schemaRegistryService Schema registry to fetch schemas
   * @param dbusEventBuffer An event buffer to which the producer can write/append events.
   * @param statsCollector Reporting stats
   * @param maxScnReaderWriters To read/write the maxScn from maxScn file
   * @throws DatabusException
   */
  public GoldenGateEventProducer(
      PhysicalSourceStaticConfig pConfig,
      SchemaRegistryService schemaRegistryService,
      DbusEventBufferAppendable dbusEventBuffer,
      DbusEventsStatisticsCollector statsCollector,
      MaxSCNReaderWriter maxScnReaderWriters)
      throws DatabusException {
    super(dbusEventBuffer, maxScnReaderWriters, pConfig, null);
    _pConfig = pConfig;
    _schemaRegistryService = schemaRegistryService;
    _statsCollector = statsCollector;
    _currentState = State.INIT;
    _partitionFunctionHashMap = new HashMap<Integer, PartitionFunction>();
    _eventsLog = Logger.getLogger("com.linkedin.databus2.producers.db.events." + pConfig.getName());

    if (_pConfig != null) {
      long eventRatePerSec = pConfig.getEventRatePerSec();
      long maxThrottleDurationInSecs = pConfig.getMaxThrottleDurationInSecs();

      if ((eventRatePerSec > 0) && (maxThrottleDurationInSecs > 0)) {
        _rc = new RateControl(eventRatePerSec, maxThrottleDurationInSecs);
      } else {
        // Disable rate control
        _rc = new RateControl(Long.MIN_VALUE, Long.MIN_VALUE);
      }
    }

    final String MODULE = GoldenGateEventProducer.class.getName();
    _log = Logger.getLogger(MODULE + "." + getName());

    // Create a hashmap for logical source id ==> PartitionFunction, this will be used as the
    // logical partition Id for the event creation
    // also create a list(map) of MonitoredSourceInfo objects to monitor GGEventProducer progress
    for (int i = 0; i < _pConfig.getSources().length; i++) {
      LogicalSourceStaticConfig logicalSourceStaticConfig = _pConfig.getSources()[i];
      GGMonitoredSourceInfo source =
          buildGGMonitoredSourceInfo(logicalSourceStaticConfig, _pConfig);
      _monitoredSources.put(source.getSourceId(), source);
    }

    // get one fake global source for total stats
    LogicalSourceStaticConfig logicalSourceStaticConfig =
        new LogicalSourceStaticConfig(
            GLOBAL_SOURCE_ID,
            _pConfig.getName(),
            "",
            "constant:1",
            (short) 0,
            false,
            null,
            null,
            null);
    GGMonitoredSourceInfo source = buildGGMonitoredSourceInfo(logicalSourceStaticConfig, _pConfig);
    _monitoredSources.put(source.getSourceId(), source);

    // create stats collector for parser
    _ggParserStats = new GGParserStatistics(_pConfig.getName());
    registerParserMbean(_ggParserStats);
  }
コード例 #2
0
  private void registerParserMbean(GGParserStatisticsMBean parserBean) throws DatabusException {
    try {
      Hashtable<String, String> props = new Hashtable<String, String>();
      props.put("type", "GGParserStatistics");
      props.put("name", _pConfig.getName());
      ObjectName objectName = new ObjectName(ServerContainer.JMX_DOMAIN, props);

      if (_mbeanServer.isRegistered(objectName)) {
        _log.warn("Unregistering old ggparser statistics mbean: " + objectName);
        _mbeanServer.unregisterMBean(objectName);
      }

      _mbeanServer.registerMBean(parserBean, objectName);
      _log.info("Registered gg-parser statistics mbean: " + objectName);
      _registeredMbeans.add(objectName);
    } catch (Exception ex) {
      _log.error(
          "Failed to register the GGparser statistics mbean for db = "
              + _pConfig.getName()
              + " due to an exception.",
          ex);
      throw new DatabusException("Failed to initialize GGparser statistics mbean.", ex);
    }
  }
コード例 #3
0
  /**
   * @param dbUpdates The dbUpdates present in the current transaction
   * @param ti The meta information about the transaction. (See TransactionInfo class for more
   *     details).
   * @throws DatabusException
   * @throws UnsupportedKeyException
   */
  protected void addEventToBuffer(
      List<TransactionState.PerSourceTransactionalUpdate> dbUpdates, TransactionInfo ti)
      throws DatabusException, UnsupportedKeyException {
    if (dbUpdates.size() == 0) throw new DatabusException("Cannot handle empty dbUpdates");

    long scn = ti.getScn();
    long timestamp = ti.getTransactionTimeStampNs();
    EventSourceStatistics globalStats = getSource(GLOBAL_SOURCE_ID).getStatisticsBean();

    /**
     * We skip the start scn of the relay, we have already added a EOP for this SCN in the buffer.
     * Why is this not a problem ? There are two cases: 1. When we use the earliest/latest scn if
     * there is no maxScn (We don't really have a start point). So it's really OK to miss the first
     * event. 2. If it's the maxSCN, then event was already seen by the relay.
     */
    if (scn == _startPrevScn.get()) {
      _log.info("Skipping this transaction, EOP already send for this event");
      return;
    }

    getEventBuffer().startEvents();

    int eventsInTransactionCount = 0;

    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();

    for (int i = 0; i < dbUpdates.size(); ++i) {
      GenericRecord record = null;
      TransactionState.PerSourceTransactionalUpdate perSourceUpdate = dbUpdates.get(i);
      short sourceId = (short) perSourceUpdate.getSourceId();
      // prepare stats collection per source
      EventSourceStatistics perSourceStats = getSource(sourceId).getStatisticsBean();

      Iterator<DbUpdateState.DBUpdateImage> dbUpdateIterator =
          perSourceUpdate.getDbUpdatesSet().iterator();
      int eventsInDbUpdate = 0;
      long dbUpdatesEventsSize = 0;
      long startDbUpdatesMs = System.currentTimeMillis();

      while (dbUpdateIterator
          .hasNext()) // TODO verify if there is any case where we need to rollback.
      {
        DbUpdateState.DBUpdateImage dbUpdate = dbUpdateIterator.next();

        // Construct the Databus Event key, determine the key type and construct the key
        Object keyObj = obtainKey(dbUpdate);
        DbusEventKey eventKey = new DbusEventKey(keyObj);

        // Get the logicalparition id
        PartitionFunction partitionFunction = _partitionFunctionHashMap.get((int) sourceId);
        short lPartitionId = partitionFunction.getPartition(eventKey);

        record = dbUpdate.getGenericRecord();
        // Write the event to the buffer
        if (record == null)
          throw new DatabusException("Cannot write event to buffer because record = " + record);

        if (record.getSchema() == null)
          throw new DatabusException("The record does not have a schema (null schema)");

        try {
          // Collect stats on number of dbUpdates for one source
          eventsInDbUpdate++;

          // Count of all the events in the current transaction
          eventsInTransactionCount++;
          // Serialize the row
          ByteArrayOutputStream bos = new ByteArrayOutputStream();
          Encoder encoder = new BinaryEncoder(bos);
          GenericDatumWriter<GenericRecord> writer =
              new GenericDatumWriter<GenericRecord>(record.getSchema());
          writer.write(record, encoder);
          byte[] serializedValue = bos.toByteArray();

          // Get the md5 for the schema
          SchemaId schemaId = SchemaId.createWithMd5(dbUpdate.getSchema());

          // Determine the operation type and convert to dbus opcode
          DbusOpcode opCode;
          if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.INSERT
              || dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.UPDATE) {
            opCode = DbusOpcode.UPSERT;
            if (_log.isDebugEnabled())
              _log.debug("The event with scn " + scn + " is INSERT/UPDATE");
          } else if (dbUpdate.getOpType() == DbUpdateState.DBUpdateImage.OpType.DELETE) {
            opCode = DbusOpcode.DELETE;
            if (_log.isDebugEnabled()) _log.debug("The event with scn " + scn + " is DELETE");
          } else {
            throw new DatabusException("Unknown opcode from dbUpdate for event with scn:" + scn);
          }

          // Construct the dbusEvent info
          DbusEventInfo dbusEventInfo =
              new DbusEventInfo(
                  opCode,
                  scn,
                  (short) _pConfig.getId(),
                  lPartitionId,
                  timestamp,
                  sourceId,
                  schemaId.getByteArray(),
                  serializedValue,
                  false,
                  false);
          dbusEventInfo.setReplicated(dbUpdate.isReplicated());

          perSourceStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);
          globalStats.addEventCycle(1, ti.getTransactionTimeRead(), serializedValue.length, scn);

          long tsEnd = System.currentTimeMillis();
          perSourceStats.addTimeOfLastDBAccess(tsEnd);
          globalStats.addTimeOfLastDBAccess(tsEnd);

          // Append to the event buffer
          getEventBuffer().appendEvent(eventKey, dbusEventInfo, _statsCollector);
          _rc.incrementEventCount();
          dbUpdatesEventsSize += serializedValue.length;
        } catch (IOException io) {
          perSourceStats.addError();
          globalStats.addEmptyEventCycle();
          _log.error("Cannot create byte stream payload: " + dbUpdates.get(i).getSourceId());
        }
      }
      long endDbUpdatesMs = System.currentTimeMillis();
      long dbUpdatesElapsedTimeMs = endDbUpdatesMs - startDbUpdatesMs;

      // Log Event Summary at logical source level
      EventReaderSummary summary =
          new EventReaderSummary(
              sourceId,
              _monitoredSources.get(sourceId).getSourceName(),
              scn,
              eventsInDbUpdate,
              dbUpdatesEventsSize,
              -1L /* Not supported */,
              dbUpdatesElapsedTimeMs,
              timestamp,
              timestamp,
              -1L /* Not supported */);
      if (_eventsLog.isInfoEnabled()) {
        _eventsLog.info(summary.toString());
      }
      summaries.add(summary);

      if (_log.isDebugEnabled())
        _log.debug("There are " + eventsInDbUpdate + " events seen in the current dbUpdate");
    }

    // update stats
    _ggParserStats.addTransactionInfo(ti, eventsInTransactionCount);

    // Log Event Summary at Physical source level
    ReadEventCycleSummary summary =
        new ReadEventCycleSummary(
            _pConfig.getName(),
            summaries,
            scn,
            -1 /* Overall time including query time not calculated */);

    if (_eventsLog.isInfoEnabled()) {
      _eventsLog.info(summary.toString());
    }

    _log.info("Writing " + eventsInTransactionCount + " events from transaction with scn: " + scn);
    if (scn <= 0)
      throw new DatabusException(
          "Unable to write events to buffer because of negative/zero scn: " + scn);

    getEventBuffer().endEvents(scn, _statsCollector);
    _scn.set(scn);

    if (getMaxScnReaderWriter() != null) {
      try {
        getMaxScnReaderWriter().saveMaxScn(_scn.get());
      } catch (DatabusException e) {
        _log.error("Cannot save scn = " + _scn + " for physical source = " + getName(), e);
      }
    }
  }
コード例 #4
0
 /** Returns the name of the source for which this relay is configured */
 @Override
 public String getName() {
   return (_pConfig != null) ? _pConfig.getName() : "NONE";
 }