/**
* Create the table definition to export to, removing any prior table. By specifying
* ColumnGenerator arguments, you can add extra columns to the table of arbitrary type.
*/
public void createTable(ColumnGenerator... extraColumns) throws SQLException {
Connection conn = getConnection();
PreparedStatement statement =
conn.prepareStatement(
getDropTableStatement(getTableName()),
ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
try {
statement.executeUpdate();
conn.commit();
} finally {
statement.close();
}
StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE ");
sb.append(getTableName());
sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
int colNum = 0;
for (ColumnGenerator gen : extraColumns) {
sb.append(", " + forIdx(colNum++) + " " + gen.getType());
}
sb.append(")");
statement =
conn.prepareStatement(
sb.toString(), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
try {
statement.executeUpdate();
conn.commit();
} finally {
statement.close();
}
}
private Record getOdpsRecord(
TableTunnel.UploadSession uploadSession, int idx, ColumnGenerator... extraCols)
throws ParseException {
Record record = uploadSession.newRecord();
record.setBigint("id", (long) idx);
record.setString("msg", getMsgPrefix() + idx);
int colNum = 0;
for (ColumnGenerator generator : extraCols) {
String field = forIdx(colNum++);
String fieldValue = generator.getExportText(idx);
switch (generator.getOdpsType()) {
case STRING:
record.setString(field, fieldValue);
break;
case BIGINT:
record.setBigint(field, Long.parseLong(fieldValue));
break;
case DATETIME:
String dateFormat = generator.getDateFormat();
record.setDatetime(field, new SimpleDateFormat(dateFormat).parse(fieldValue));
break;
case DOUBLE:
record.setDouble(field, Double.parseDouble(fieldValue));
break;
case DECIMAL:
record.setDecimal(field, new BigDecimal(fieldValue));
break;
default:
throw new RuntimeException("Unknown column type: " + generator.getOdpsType());
}
}
return record;
}
/**
* Verify that for the max and min values of the 'id' column, the values for a given column meet
* the expected values.
*/
private void assertColMinAndMax(String colName, ColumnGenerator generator) throws SQLException {
Connection conn = getConnection();
int minId = getMinRowId(conn);
int maxId = getMaxRowId(conn);
LOG.info("Checking min/max for column " + colName + " with type " + generator.getType());
String expectedMin = generator.getVerifyText(minId);
String expectedMax = generator.getVerifyText(maxId);
assertColValForRowId(minId, colName, expectedMin);
assertColValForRowId(maxId, colName, expectedMax);
}
protected void createOdpsTable(
String tableName, String[] partitionCols, ColumnGenerator... extraCols) throws OdpsException {
TableSchema schema = new TableSchema();
schema.addColumn(new Column("id", OdpsType.BIGINT));
schema.addColumn(new Column("msg", OdpsType.STRING));
int colNum = 0;
for (ColumnGenerator generator : extraCols) {
schema.addColumn(new Column(forIdx(colNum++), generator.getOdpsType()));
}
if (partitionCols != null) {
for (String partition : partitionCols) {
schema.addPartitionColumn(new Column(partition, OdpsType.STRING));
}
}
odps.tables().create(tableName, schema);
}
protected String getRecordLine(int recordNum, ColumnGenerator... extraCols) {
String idStr = Integer.toString(recordNum);
StringBuilder sb = new StringBuilder();
sb.append(idStr);
sb.append("\t");
sb.append(getMsgPrefix());
sb.append(idStr);
for (ColumnGenerator gen : extraCols) {
sb.append("\t");
sb.append(gen.getExportText(recordNum));
}
sb.append("\n");
return sb.toString();
}
@Override
public void init(String[] args) {
super.init(args);
if (arguments.length == 1) {
model = arguments[0];
}
model = model.trim();
fullIp = model.split(":");
sepIp = fullIp[0].split("\\.");
for (int i = 0; i < sepIp.length; i++) {
String[] token = (sepIp[i]).split("\\-");
min[i] = 1;
max[i] = 255;
if (token.length == 1 && !token[0].startsWith("*")) {
min[i] = Integer.valueOf(token[0]);
max[i] = min[i];
}
if (token.length == 2) {
min[i] = Integer.valueOf(token[0]);
max[i] = Integer.valueOf(token[1]);
}
}
if (fullIp.length == 2) {
minPort = 1;
maxPort = 65535;
String[] token = fullIp[1].split("\\-");
if (token.length == 1 && !token[0].startsWith("*")) {
minPort = Integer.valueOf(token[0]);
maxPort = minPort;
}
if (token.length == 2) {
minPort = Integer.valueOf(token[0]);
maxPort = Integer.valueOf(token[1]);
}
}
}