/**
   * Create the table definition to export to, removing any prior table. By specifying
   * ColumnGenerator arguments, you can add extra columns to the table of arbitrary type.
   */
  public void createTable(ColumnGenerator... extraColumns) throws SQLException {
    Connection conn = getConnection();
    PreparedStatement statement =
        conn.prepareStatement(
            getDropTableStatement(getTableName()),
            ResultSet.TYPE_FORWARD_ONLY,
            ResultSet.CONCUR_READ_ONLY);
    try {
      statement.executeUpdate();
      conn.commit();
    } finally {
      statement.close();
    }

    StringBuilder sb = new StringBuilder();
    sb.append("CREATE TABLE ");
    sb.append(getTableName());
    sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
    int colNum = 0;
    for (ColumnGenerator gen : extraColumns) {
      sb.append(", " + forIdx(colNum++) + " " + gen.getType());
    }
    sb.append(")");

    statement =
        conn.prepareStatement(
            sb.toString(), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
    try {
      statement.executeUpdate();
      conn.commit();
    } finally {
      statement.close();
    }
  }
 private Record getOdpsRecord(
     TableTunnel.UploadSession uploadSession, int idx, ColumnGenerator... extraCols)
     throws ParseException {
   Record record = uploadSession.newRecord();
   record.setBigint("id", (long) idx);
   record.setString("msg", getMsgPrefix() + idx);
   int colNum = 0;
   for (ColumnGenerator generator : extraCols) {
     String field = forIdx(colNum++);
     String fieldValue = generator.getExportText(idx);
     switch (generator.getOdpsType()) {
       case STRING:
         record.setString(field, fieldValue);
         break;
       case BIGINT:
         record.setBigint(field, Long.parseLong(fieldValue));
         break;
       case DATETIME:
         String dateFormat = generator.getDateFormat();
         record.setDatetime(field, new SimpleDateFormat(dateFormat).parse(fieldValue));
         break;
       case DOUBLE:
         record.setDouble(field, Double.parseDouble(fieldValue));
         break;
       case DECIMAL:
         record.setDecimal(field, new BigDecimal(fieldValue));
         break;
       default:
         throw new RuntimeException("Unknown column type: " + generator.getOdpsType());
     }
   }
   return record;
 }
Esempio n. 3
0
  /**
   * Verify that for the max and min values of the 'id' column, the values for a given column meet
   * the expected values.
   */
  private void assertColMinAndMax(String colName, ColumnGenerator generator) throws SQLException {
    Connection conn = getConnection();
    int minId = getMinRowId(conn);
    int maxId = getMaxRowId(conn);

    LOG.info("Checking min/max for column " + colName + " with type " + generator.getType());

    String expectedMin = generator.getVerifyText(minId);
    String expectedMax = generator.getVerifyText(maxId);

    assertColValForRowId(minId, colName, expectedMin);
    assertColValForRowId(maxId, colName, expectedMax);
  }
 protected void createOdpsTable(
     String tableName, String[] partitionCols, ColumnGenerator... extraCols) throws OdpsException {
   TableSchema schema = new TableSchema();
   schema.addColumn(new Column("id", OdpsType.BIGINT));
   schema.addColumn(new Column("msg", OdpsType.STRING));
   int colNum = 0;
   for (ColumnGenerator generator : extraCols) {
     schema.addColumn(new Column(forIdx(colNum++), generator.getOdpsType()));
   }
   if (partitionCols != null) {
     for (String partition : partitionCols) {
       schema.addPartitionColumn(new Column(partition, OdpsType.STRING));
     }
   }
   odps.tables().create(tableName, schema);
 }
Esempio n. 5
0
  protected String getRecordLine(int recordNum, ColumnGenerator... extraCols) {
    String idStr = Integer.toString(recordNum);
    StringBuilder sb = new StringBuilder();

    sb.append(idStr);
    sb.append("\t");
    sb.append(getMsgPrefix());
    sb.append(idStr);
    for (ColumnGenerator gen : extraCols) {
      sb.append("\t");
      sb.append(gen.getExportText(recordNum));
    }
    sb.append("\n");

    return sb.toString();
  }
Esempio n. 6
0
  @Override
  public void init(String[] args) {
    super.init(args);
    if (arguments.length == 1) {
      model = arguments[0];
    }
    model = model.trim();
    fullIp = model.split(":");
    sepIp = fullIp[0].split("\\.");

    for (int i = 0; i < sepIp.length; i++) {
      String[] token = (sepIp[i]).split("\\-");
      min[i] = 1;
      max[i] = 255;
      if (token.length == 1 && !token[0].startsWith("*")) {
        min[i] = Integer.valueOf(token[0]);
        max[i] = min[i];
      }
      if (token.length == 2) {
        min[i] = Integer.valueOf(token[0]);
        max[i] = Integer.valueOf(token[1]);
      }
    }
    if (fullIp.length == 2) {
      minPort = 1;
      maxPort = 65535;
      String[] token = fullIp[1].split("\\-");
      if (token.length == 1 && !token[0].startsWith("*")) {
        minPort = Integer.valueOf(token[0]);
        maxPort = minPort;
      }
      if (token.length == 2) {
        minPort = Integer.valueOf(token[0]);
        maxPort = Integer.valueOf(token[1]);
      }
    }
  }