Example #1
0
 public Object orToMysqlType(Column column) throws Exception {
   if (column instanceof BitColumn) {
     // This is in byte order
     BitColumn byteColumn = (BitColumn) column;
     byte[] byteArray = byteColumn.getValue();
     return new String(byteArray);
   } else if (column instanceof BlobColumn) {
     BlobColumn blobColumn = (BlobColumn) column;
     byte[] byteArray = blobColumn.getValue();
     return new String(byteArray);
   } else if (column instanceof DateColumn) {
     DateColumn dateColumn = (DateColumn) column;
     Date date = dateColumn.getValue();
     return new java.sql.Date(date.getTime());
   } else if (column instanceof DatetimeColumn) {
     DatetimeColumn dateTimeColumn = (DatetimeColumn) column;
     Date date = dateTimeColumn.getValue();
     /**
      * Bug in OR for DateTIme and Time data-types. MilliSeconds is not available for these columns
      * but is set with currentMillis() wrongly.
      */
     return new java.sql.Timestamp((date.getTime() / 1000) * 1000);
   } else if (column instanceof DecimalColumn) {
     DecimalColumn decimalColumn = (DecimalColumn) column;
     return decimalColumn.getValue();
   } else if (column instanceof DoubleColumn) {
     DoubleColumn doubleColumn = (DoubleColumn) column;
     return doubleColumn.getValue();
   } else if (column instanceof EnumColumn) {
     EnumColumn enumColumn = (EnumColumn) column;
     return enumColumn.getValue();
   } else if (column instanceof FloatColumn) {
     FloatColumn floatColumn = (FloatColumn) column;
     return floatColumn.getValue();
   } else if (column instanceof Int24Column) {
     Int24Column intColumn = (Int24Column) column;
     return intColumn.getValue();
   } else if (column instanceof LongColumn) {
     LongColumn longColumn = (LongColumn) column;
     return longColumn.getValue();
   } else if (column instanceof LongLongColumn) {
     LongLongColumn longLongColumn = (LongLongColumn) column;
     return longLongColumn.getValue();
   } else if (column instanceof NullColumn) {
     return null;
   } else if (column instanceof SetColumn) {
     SetColumn setColumn = (SetColumn) column;
     return setColumn.getValue();
   } else if (column instanceof ShortColumn) {
     ShortColumn shortColumn = (ShortColumn) column;
     return shortColumn.getValue();
   } else if (column instanceof StringColumn) {
     StringColumn stringColumn = (StringColumn) column;
     return new String(stringColumn.getValue(), Charset.defaultCharset());
   } else if (column instanceof TimeColumn) {
     TimeColumn timeColumn = (TimeColumn) column;
     Time time = timeColumn.getValue();
     /**
      * There is a bug in OR where instead of using the default year as 1970, it is using 0070.
      * This is a temporary measure to resolve it by working around at this layer. The value
      * obtained from OR is subtracted from "0070-00-01 00:00:00"
      */
     Calendar c = Calendar.getInstance();
     c.set(70, 0, 1, 0, 0, 0);
     /**
      * round off the milli-seconds as TimeColumn type has only seconds granularity but Calendar
      * implementation includes milli-second (System.currentTimeMillis() at the time of
      * instantiation)
      */
     long rawVal = (c.getTimeInMillis() / 1000) * 1000;
     long val2 = (time.getTime() / 1000) * 1000;
     return new java.sql.Time(val2 - rawVal);
   } else if (column instanceof TimestampColumn) {
     TimestampColumn timeStampColumn = (TimestampColumn) column;
     return timeStampColumn.getValue();
   } else if (column instanceof TinyColumn) {
     TinyColumn tinyColumn = (TinyColumn) column;
     return tinyColumn.getValue();
   } else if (column instanceof YearColumn) {
     YearColumn yearColumn = (YearColumn) column;
     return yearColumn.getValue();
   } else {
     String message =
         "Unknown MySQL type in the event" + column.getClass() + " Object = " + column;
     LOGGER.error(message);
     throw new RuntimeException(message);
   }
 }
  @Override
  public List<SourceRecord> poll() throws InterruptedException {

    try {
      MaxwellAbstractRowsEvent event = replicator.getEvent();
      this.maxwellContext.ensurePositionThread();

      if (event == null) {
        return null;
      }

      if (event.getTable().getDatabase().getName().equals("maxwell")) {
        return null;
      }

      String databaseName = event.getDatabase().getName();
      String tableName = event.getTable().getName();

      String topicName = databaseName + "." + tableName;

      ArrayList<SourceRecord> records = new ArrayList<>();

      Table table = event.getTable();

      List<Row> rows = event.filteredRows();
      // databaseName.tableName
      // create schema for primary key
      Schema pkSchema = DataConverter.convertPrimaryKeySchema(table);

      List<Struct> primaryKeys = new ArrayList<Struct>();

      for (Row row : rows) {
        // make primary key schema
        Struct pkStruct = new Struct(pkSchema);

        for (String pk : table.getPKList()) {
          int idx = table.findColumnIndex(pk);

          Column column = row.getColumns().get(idx);
          ColumnDef def = table.getColumnList().get(idx);

          switch (def.getType()) {
            case "int":
              IntColumnDef intDef = (IntColumnDef) def;
              Long l = intDef.toLong(column.getValue());
              pkStruct.put(pk, l.intValue());
              break;
            default:
              throw new RuntimeException("unsupported type");
          }
        }
        primaryKeys.add(pkStruct);
      }

      Iterator<String> jsonIter = event.toJSONStrings().iterator();
      //            Iterator<String> keysIter = event.getPKStrings().iterator();

      Iterator<Struct> pkIter = primaryKeys.iterator();

      while (jsonIter.hasNext() && pkIter.hasNext()) {
        String json = jsonIter.next();
        Struct key = pkIter.next();

        System.out.print("got a maxwell event!");
        System.out.println(json);
        SourceRecord rec =
            new SourceRecord(
                sourcePartition(),
                sourceOffset(event),
                topicName,
                null, // partition
                pkSchema,
                key,
                VALUE_SCHEMA,
                json);
        records.add(rec);
      }
      return records;

      // return records;
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
  }