private Record getOdpsRecord(
TableTunnel.UploadSession uploadSession, int idx, ColumnGenerator... extraCols)
throws ParseException {
Record record = uploadSession.newRecord();
record.setBigint("id", (long) idx);
record.setString("msg", getMsgPrefix() + idx);
int colNum = 0;
for (ColumnGenerator generator : extraCols) {
String field = forIdx(colNum++);
String fieldValue = generator.getExportText(idx);
switch (generator.getOdpsType()) {
case STRING:
record.setString(field, fieldValue);
break;
case BIGINT:
record.setBigint(field, Long.parseLong(fieldValue));
break;
case DATETIME:
String dateFormat = generator.getDateFormat();
record.setDatetime(field, new SimpleDateFormat(dateFormat).parse(fieldValue));
break;
case DOUBLE:
record.setDouble(field, Double.parseDouble(fieldValue));
break;
case DECIMAL:
record.setDecimal(field, new BigDecimal(fieldValue));
break;
default:
throw new RuntimeException("Unknown column type: " + generator.getOdpsType());
}
}
return record;
}
protected void createOdpsTableWithRecords(
String tableName, String[] partitionCols, int numRecords, ColumnGenerator... extraCols)
throws OdpsException, IOException, ParseException {
createOdpsTable(tableName, partitionCols, extraCols);
TableTunnel tunnel = new TableTunnel(odps);
PartitionSpec partitionSpec = getSimplePartitionSpec(partitionCols);
TableTunnel.UploadSession uploadSession =
partitionSpec == null
? tunnel.createUploadSession(PROJECT, tableName)
: tunnel.createUploadSession(PROJECT, tableName, partitionSpec);
RecordWriter recordWriter = uploadSession.openRecordWriter(0);
for (int i = 0; i < numRecords; i++) {
Record record = getOdpsRecord(uploadSession, i, extraCols);
recordWriter.write(record);
}
recordWriter.close();
uploadSession.commit(new Long[] {0L});
}