/*
   * Test test that event ID calls work as expected
   */
  public void testExtractorEventID() throws Exception {

    RawExtractor extractor = (RawExtractor) PluginLoader.load(DummyExtractor.class.getName());

    extractor.configure(null);
    extractor.prepare(null);

    DBMSEvent event = extractor.extract();
    String currentEventId = extractor.getCurrentResourceEventId();
    Assert.assertEquals(event.getEventId(), currentEventId);

    event = extractor.extract();
    Assert.assertTrue(event.getEventId().compareTo(currentEventId) > 0);

    currentEventId = extractor.getCurrentResourceEventId();
    Assert.assertTrue(event.getEventId().compareTo(currentEventId) == 0);

    extractor.release(null);
  }
  /*
   * Test that dummy extractor works like expected,
   */
  public void testExtractorBasic() throws Exception {

    RawExtractor extractor = (RawExtractor) PluginLoader.load(DummyExtractor.class.getName());

    extractor.configure(null);
    extractor.prepare(null);

    DBMSEvent event = extractor.extract();
    Assert.assertEquals(event.getEventId(), "0");
    event = extractor.extract();
    Assert.assertEquals(event.getEventId(), "1");

    extractor.setLastEventId("0");
    event = extractor.extract();
    Assert.assertEquals(event.getEventId(), "1");

    extractor.setLastEventId(null);
    event = extractor.extract();
    Assert.assertEquals(event.getEventId(), "0");

    for (Integer i = 1; i < 5; ++i) {
      event = extractor.extract();
      Assert.assertEquals(event.getEventId(), i.toString());
    }

    event = extractor.extract("0");
    Assert.assertEquals(event.getEventId(), "0");

    event = extractor.extract("4");
    Assert.assertEquals(event.getEventId(), "4");

    event = extractor.extract("5");
    Assert.assertEquals(event, null);

    extractor.release(null);
  }
  /**
   * Post all committed changes to given queue
   *
   * @param q Queue to post
   * @param minSCN Minimal SCN among all open transactions
   * @param skipSeq If only part of the transaction should be posted, this is the last seq to skip
   * @param lastObsoletePlogSeq Last obsolete plog sequence (min() over all open transactions - 1)
   * @return lastProcessedEventId
   */
  public String pushContentsToQueue(
      BlockingQueue<DBMSEvent> q, long minSCN, int transactionFragSize, long lastObsoletePlogSeq)
      throws UnsupportedEncodingException, ReplicatorException, SerialException,
          InterruptedException, SQLException {
    if (logger.isDebugEnabled()) {
      logger.debug("pushContentsToQueue: " + this.XID + " (DML:" + this.transactionIsDML + ")");
    }

    if (transactionIsDML) {
      int count = 0;
      PlogLCR lastLCR = null;
      String lastProcessedEventId = null;
      ArrayList<DBMSData> data = new ArrayList<DBMSData>();
      int fragSize = 0;

      LargeObjectScanner<PlogLCR> scanner = null;
      try {
        scanner = LCRList.scanner();
        while (scanner.hasNext()) {
          PlogLCR LCR = scanner.next();
          if (LCR.type == PlogLCR.ETYPE_LCR_DATA) {
              /* add only data */
            if (LCR.subtype == PlogLCR.ESTYPE_LCR_DDL)
              throw new ReplicatorException(
                  "Internal corruption: DDL statement in a DML transaction.");
            if (transactionFragSize > 0 && fragSize >= transactionFragSize) {
              logger.debug("Fragmenting");
              // Time to fragment
              DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, false, commitTime);

              // Set metadata for the transaction. Source is
              // Oracle, we normalize time data to GMT, and
              // strings are in UTF8.
              event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
              event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
              event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
              q.put(event);

              // Clear array for next fragment.
              data = new ArrayList<DBMSData>();
              fragSize = 0;
            }

            count++;
            fragSize++;
            if (count > skipSeq) {
              LCR.eventId =
                  ""
                      + commitSCN
                      + "#"
                      + XID
                      + "#"
                      + count
                      + "#"
                      + minSCN
                      + "#"
                      + lastObsoletePlogSeq;
              lastLCR = LCR;

              data.add(convertLCRtoDBMSDataDML(LCR));
              lastProcessedEventId = LCR.eventId;

              {
                PlogLCR r2 = LCR;
                if (logger.isDebugEnabled()) {
                  logger.debug(
                      "LCR: "
                          + r2.type
                          + "."
                          + r2.subtype
                          + ":"
                          + r2.typeAsString()
                          + ", XID="
                          + r2.XID
                          + ", LCRid="
                          + r2.LCRid
                          + " eventId="
                          + r2.eventId);
                  logger.debug("EventId#1 set to " + r2.eventId);
                }
              }
            }
          } else {
            throw new RuntimeException("Type " + LCR.type + " in queue, should be only data");
          }
        }
      } catch (IOException e) {
        logger.error(
            "Transaction scanner error occured: XID="
                + this.XID
                + " key="
                + LCRList.getKey()
                + " cache="
                + LCRList.getByteCache().toString());
        throw new RuntimeException("Unable to read transaction from cache: " + this.XID);
      } finally {
        if (scanner != null) scanner.close();
      }
      if (lastLCR != null) {
        // Last LCR set = transaction is not empty
        // Mark last LCR as "LAST", so we know transaction is complete
        lastLCR.eventId =
            "" + commitSCN + "#" + XID + "#" + "LAST" + "#" + minSCN + "#" + lastObsoletePlogSeq;
        if (logger.isDebugEnabled()) {
          logger.debug("EventId#2 set to " + lastLCR.eventId);
        }

        DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, true, commitTime);

        // Set metadata for the transaction. Source is Oracle, we
        // normalize time data to GMT, and strings are in UTF8.
        event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
        event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
        event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");

        q.put(event);
      }

      return lastProcessedEventId;
    } else {
      ArrayList<DBMSData> data = new ArrayList<DBMSData>();
      PlogLCR lastLCR = null;
      String lastProcessedEventId = null;

      LargeObjectScanner<PlogLCR> scanner = null;
      try {
        scanner = LCRList.scanner();
        while (scanner.hasNext()) {
          PlogLCR LCR = scanner.next();
          if (LCR.type == PlogLCR.ETYPE_LCR_DATA) {
              /* add only data */
            if (LCR.subtype != PlogLCR.ESTYPE_LCR_DDL) {
              throw new ReplicatorException(
                  "Internal corruption: DML statement in a DDL transaction.");
            }
            LCR.eventId =
                ""
                    + commitSCN
                    + "#"
                    + XID
                    + "#"
                    + "LAST"
                    + "#"
                    + minSCN
                    + "#"
                    + lastObsoletePlogSeq;
            data.add(convertLCRtoDBMSDataDDL(LCR));
            lastLCR = LCR;
            lastProcessedEventId = LCR.eventId;
            if (logger.isDebugEnabled()) {
              logger.info("DDL: [" + LCR.currentSchema + "][" + LCR.eventId + "]: " + LCR.SQLText);
            }
          } else {
            throw new RuntimeException("Type " + LCR.type + " in queue, should be only data");
          }
        }
      } catch (IOException e) {
        logger.error(
            "Transaction scanner error occured: XID="
                + this.XID
                + " key="
                + LCRList.getKey()
                + " cache="
                + LCRList.getByteCache().toString());
        throw new RuntimeException("Unable to read transaction from cache: " + this.XID);
      } finally {
        if (scanner != null) scanner.close();
      }

      if (!data.isEmpty()) {
        DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, true, commitTime);
        event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
        event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
        event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
        q.put(event);
      }
      return lastProcessedEventId;
    }
  }
  /**
   * {@inheritDoc}
   *
   * @see com.continuent.tungsten.replicator.extractor.RawExtractor#extract()
   */
  @Override
  public DBMSEvent extract() throws ReplicatorException, InterruptedException {
    ArrayList<DBMSData> data = new ArrayList<DBMSData>();
    long maxSCN = -1;
    Timestamp sourceTStamp = null;

    boolean noData = true;

    try {
      if (logger.isDebugEnabled()) logger.debug("Extending Window");

      executeQuery(
          "BEGIN DBMS_CDC_SUBSCRIBE.EXTEND_WINDOW(subscription_name => 'TUNGSTEN_PUB');END;",
          false);

      if (logger.isDebugEnabled())
        logger.debug("Handling " + subscriberViews.keySet().size() + "views");
      for (String view : subscriberViews.keySet()) {
        OracleCDCSource cdcSource = subscriberViews.get(view);

        Statement stmt = connection.getConnection().createStatement();

        String statement;
        if (lastSCN != null)
          statement =
              "SELECT "
                  + cdcSource.getPublication(view).getColumnList()
                  + " , CSCN$, COMMIT_TIMESTAMP$, OPERATION$"
                  + " from "
                  + view
                  + " where cscn$ > "
                  + lastSCN
                  + "  order by cscn$, rsid$";
        else
          statement =
              "SELECT "
                  + cdcSource.getPublication(view).getColumnList()
                  + " , CSCN$, COMMIT_TIMESTAMP$, OPERATION$"
                  + " from "
                  + view
                  + "  order by cscn$, rsid$";

        resultset = stmt.executeQuery(statement);

        int userColumns = cdcSource.getPublication(view).getColumnsCount();

        if (logger.isDebugEnabled()) logger.debug("Running " + statement);
        OneRowChange updateRowChange = null;
        OneRowChange oneRowChange = null;

        while (resultset.next()) {
          noData = false;
          long currentSCN = resultset.getLong("CSCN$");
          if (maxSCN < currentSCN) maxSCN = currentSCN;

          if (sourceTStamp == null) sourceTStamp = resultset.getTimestamp("COMMIT_TIMESTAMP$");

          if (logger.isDebugEnabled()) {
            logger.debug("Receiving data");
            StringBuffer buffer = new StringBuffer();

            for (int i = 1; i <= resultset.getMetaData().getColumnCount(); i++) {
              if (buffer.length() > 0) buffer.append('\t');
              buffer.append(resultset.getString(i));
            }
            logger.debug("Received : " + buffer.toString());
          }

          String operation = resultset.getString("OPERATION$").trim();

          RowChangeData rowData = new RowChangeData();

          if (operation.equals("I")) {
            if (oneRowChange == null || !oneRowChange.getAction().equals(ActionType.INSERT)) {
              oneRowChange =
                  new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.INSERT);
              rowData.appendOneRowChange(oneRowChange);
              data.add(rowData);
            }
            parseRowEvent(oneRowChange, false, userColumns);
          } else if (operation.equals("D")) {
            if (oneRowChange == null || !oneRowChange.getAction().equals(ActionType.DELETE)) {
              oneRowChange =
                  new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.DELETE);
              rowData.appendOneRowChange(oneRowChange);
              data.add(rowData);
            }
            parseRowEvent(oneRowChange, true, userColumns);
          } else if (operation.startsWith("U")) {
            if (updateRowChange == null) {
              updateRowChange =
                  new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.UPDATE);
              rowData.appendOneRowChange(updateRowChange);
              data.add(rowData);
              if (operation.equals("UO")) {
                parseRowEvent(updateRowChange, true, userColumns);
              } else if (operation.equals("UN")) {
                parseRowEvent(updateRowChange, false, userColumns);
              }
            } else {
              if (operation.equals("UO")) {
                parseRowEvent(updateRowChange, true, userColumns);
              } else if (operation.equals("UN")) {
                parseRowEvent(updateRowChange, false, userColumns);
              }
            }
          } else {
            logger.error(
                "Unable to extract data from CDC (operation should be I, D, UO or UN - found "
                    + operation
                    + ")");
          }
        }
        resultset.close();
        resultset = null;
        stmt.close();
      }
      lastSCN = null;
    } catch (SQLException e) {
      throw new ReplicatorException(e);
    } finally {
      if (resultset != null)
        try {
          resultset.close();
        } catch (SQLException e) {
          logger.warn("Failed to close resultset");
        }
      resultset = null;
    }
    if (noData) {
      logger.warn("Retrieved empty resultset... no data available... sleeping");
      Thread.sleep(1000);
    }
    if (logger.isDebugEnabled()) logger.debug("Purging window");

    executeQuery(
        "BEGIN DBMS_CDC_SUBSCRIBE.PURGE_WINDOW(subscription_name => 'TUNGSTEN_PUB');END;", false);

    if (data.size() > 0) {
      DBMSEvent event = new DBMSEvent(String.valueOf(maxSCN), data, sourceTStamp);

      // Mark the event as coming from Oracle.
      event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);

      // Strings are converted to UTF8 rather than using bytes for this
      // extractor.
      event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");

      return event;
    } else {
      return null;
    }
  }