@Override
protected List<Account> parseItem(String[] strings) {
List<Account> result = new ArrayList<>();
List<ColumnDefinition> columnDefinitions = getColumnDefinitions();
if (strings.length == VALUE_COLMN + columnDefinitions.size()) {
String areaKey = strings[AREA_KEY];
if (isAreaKeyAcceptable(areaKey)) {
String areaLabel = strings[AREA_LABEL];
String accountKey = strings[ACCOUNT_KEY];
String accountName = strings[ACCOUNT_NAME];
if (isAreaKeyValid(areaKey) && Utils.hasText(accountKey)) {
int parsedAccountKey = Integer.parseInt(accountKey);
for (int x = 0; x < columnDefinitions.size(); x++) {
ColumnDefinition cd = columnDefinitions.get(x);
String value = strings[VALUE_COLMN + x];
LongValue convertedValue = LongValue.valueOf(value);
result.add(
new Account(
areaKey,
areaLabel,
cd.getKey(),
cd.getLabel(),
parsedAccountKey,
accountName,
convertedValue));
addLabel(parsedAccountKey, accountName);
}
}
}
}
return result;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof ColumnDefinition)) {
return false;
}
ColumnDefinition cd = (ColumnDefinition) o;
if (this.columnName.equals(cd.getColumnName()) && this.columnType == cd.getColumnType()) {
return true;
} else {
return false;
}
}
/**
* Constructor.
*
* @param table table definition
* @param data table data
* @throws MPXJException
*/
public FileRow(TableDefinition table, List<String> data) throws MPXJException {
super(new HashMap<String, Object>());
ColumnDefinition[] columns = table.getColumns();
for (int index = 0; index < columns.length; index++) {
ColumnDefinition column = columns[index];
if (index < data.size()) {
m_map.put(
column.getName(),
getColumnValue(table.getName(), column.getName(), data.get(index), column.getType()));
}
}
}
private CodeBlock buildPopulateValuesIntoCursor() {
CodeBlock.Builder builder = CodeBlock.builder();
List<ColumnDefinition> columns = schema.getColumns();
for (int i = 0; i < columns.size(); i++) {
ColumnDefinition c = columns.get(i);
RelationDefinition r = c.getRelation();
TypeName type = c.getUnboxType();
if (r == null) {
CodeBlock.Builder getCursorExpr = CodeBlock.builder();
if (Types.needsTypeAdapter(type)) {
getCursorExpr.add(
"conn.getTypeAdapterRegistry().$L($T.$L.type, $L)",
c.nullable ? "deserializeNullable" : "deserialize",
schema.getSchemaClassName(),
c.name,
cursorGetter(c, i));
} else {
getCursorExpr.add("$L", cursorGetter(c, i));
}
if (c.setter != null) {
builder.addStatement("model.$L($L)", c.setter.getSimpleName(), getCursorExpr.build());
} else {
builder.addStatement("model.$L = $L", c.name, getCursorExpr.build());
}
} else { // SingleRelation
if (c.setter != null) {
builder.addStatement(
"model.$L(new $T<>(conn, OrmaDatabase.schema$T, cursor.getLong($L)))",
c.setter.getSimpleName(),
r.relationType,
r.modelType,
i);
} else {
builder.addStatement(
"model.$L = new $T<>(conn, OrmaDatabase.schema$T, cursor.getLong($L))",
c.name,
r.relationType,
r.modelType,
i);
}
}
}
return builder.build();
}
// http://developer.android.com/intl/ja/reference/android/database/sqlite/SQLiteStatement.html
private CodeBlock buildBindArgs() {
CodeBlock.Builder builder = CodeBlock.builder();
List<ColumnDefinition> columns = schema.getColumnsWithoutAutoId();
for (int i = 0; i < columns.size(); i++) {
int n = i + 1; // bind index starts 1
ColumnDefinition c = columns.get(i);
TypeName type = c.getUnboxType();
RelationDefinition r = c.getRelation();
boolean nullable = !type.isPrimitive() && c.nullable;
if (nullable) {
builder.beginControlFlow("if (model.$L != null)", c.name);
}
if (type.equals(TypeName.BOOLEAN)) {
builder.addStatement(
"statement.bindLong($L, model.$L ? 1 : 0)", n, c.getColumnGetterExpr());
} else if (Types.looksLikeIntegerType(type)) {
builder.addStatement("statement.bindLong($L, model.$L)", n, c.getColumnGetterExpr());
} else if (Types.looksLikeFloatType(type)) {
builder.addStatement("statement.bindDouble($L, model.$L)", n, c.getColumnGetterExpr());
} else if (type.equals(Types.ByteArray)) {
builder.addStatement("statement.bindBlob($L, model.$L)", n, c.getColumnGetterExpr());
} else if (type.equals(Types.String)) {
builder.addStatement("statement.bindString($L, model.$L)", n, c.getColumnGetterExpr());
} else if (r != null && r.relationType.equals(Types.SingleRelation)) {
builder.addStatement(
"statement.bindLong($L, model.$L.getId())", n, c.getColumnGetterExpr());
} else {
builder.addStatement(
"statement.bindString($L, conn.getTypeAdapterRegistry().serialize($T.$L.type, model.$L))",
n,
schema.getSchemaClassName(),
c.name,
c.getColumnGetterExpr());
}
if (nullable) {
builder.endControlFlow();
builder.beginControlFlow("else");
builder.addStatement("statement.bindNull($L)", n);
builder.endControlFlow();
}
}
return builder.build();
}
public FieldSpec buildColumnFieldSpec(ColumnDefinition c) {
TypeName type = c.getType();
CodeBlock initializer;
if (type instanceof ParameterizedTypeName) {
initializer =
CodeBlock.builder()
.add(
"new $T($S, $T.getType(new $T<$T>(){}), $L, $L, $L, $L, $L, $L)",
c.getColumnDefType(),
c.columnName,
Types.ParameterizedTypes,
Types.TypeHolder,
type,
c.nullable,
c.primaryKey,
c.autoincrement,
c.autoId,
c.indexed,
c.unique)
.build();
} else {
initializer =
CodeBlock.builder()
.add(
"new $T($S, $T.class, $L, $L, $L, $L, $L, $L)",
c.getColumnDefType(),
c.columnName,
type,
c.nullable,
c.primaryKey,
c.autoincrement,
c.autoId,
c.indexed,
c.unique)
.build();
}
return FieldSpec.builder(c.getColumnDefType(), c.name)
.addModifiers(publicStaticFinal)
.initializer(initializer)
.build();
}
/**
* Deserialize ColumnFamilies from low-level schema representation, all of them belong to the same
* keyspace
*
* @param row
* @return map containing name of the ColumnFamily and it's metadata for faster lookup
*/
public static Map<String, CFMetaData> deserializeColumnFamilies(Row row) {
if (row.cf == null) return Collections.emptyMap();
Map<String, CFMetaData> cfms = new HashMap<String, CFMetaData>();
UntypedResultSet results =
QueryProcessor.resultify("SELECT * FROM system.schema_columnfamilies", row);
for (UntypedResultSet.Row result : results) {
CFMetaData cfm = CFMetaData.fromSchema(result);
cfms.put(cfm.cfName, cfm);
}
for (CFMetaData cfm : cfms.values()) {
Row columnRow = ColumnDefinition.readSchema(cfm.ksName, cfm.cfName);
for (ColumnDefinition cd : ColumnDefinition.fromSchema(columnRow, cfm))
cfm.column_metadata.put(cd.name, cd);
}
return cfms;
}
/**
* Gets partition where tuple has to be written. Creates the partition if necessary.
*
* @param t
* @return
* @throws IOException
*/
public synchronized Partition getPartitionForTuple(Tuple t) throws IOException {
// if(partitioningSpec==null) return
// tableDefinition.getDataDir()+"/"+tableDefinition.getName();
long time = TimeEncoding.INVALID_INSTANT;
Object value = null;
if (partitioningSpec.timeColumn != null) {
time = (Long) t.getColumn(partitioningSpec.timeColumn);
}
if (partitioningSpec.valueColumn != null) {
value = t.getColumn(partitioningSpec.valueColumn);
ColumnDefinition cd = tableDefinition.getColumnDefinition(partitioningSpec.valueColumn);
if (cd.getType() == DataType.ENUM) {
value = tableDefinition.addAndGetEnumValue(partitioningSpec.valueColumn, (String) value);
}
}
return createAndGetPartition(time, value);
}
public void renderRowValue(
RowType rowValue,
ColumnDefinition<RowType, String> columnDef,
AbstractCellView<RowType> view) {
String cellValue = columnDef.getCellValue(rowValue);
if (cellValue == null) {
view.setText("");
} else {
view.setHTML(cellValue.toString());
}
}
private String cursorGetter(ColumnDefinition column, int position) {
TypeName type = column.getUnboxType();
if (type.equals(TypeName.BOOLEAN)) {
return "cursor.getLong(" + position + ") != 0";
} else if (type.equals(TypeName.BYTE)) {
return "(byte)cursor.getShort(" + position + ")";
} else if (type.isPrimitive()) {
String s = type.toString();
return "cursor.get" + s.substring(0, 1).toUpperCase() + s.substring(1) + "(" + position + ")";
} else if (type.equals(Types.String)) {
return "cursor.getString(" + position + ")";
} else if (type.equals(Types.ByteArray)) {
return "cursor.getBlob(" + position + ")";
} else {
return "cursor.getString(" + position + ")"; // handled by type adapters
}
}
@Override
public Object clone() throws CloneNotSupportedException {
AlterTable alter = (AlterTable) super.clone();
alter.tableName = tableName;
alter.newTableName = newTableName;
alter.columnName = columnName;
alter.newColumnName = newColumnName;
if (addNewColumn != null) {
alter.addNewColumn = (ColumnDefinition) addNewColumn.clone();
}
alter.alterTableOpType = alterTableOpType;
alter.columns = columns;
alter.values = values;
alter.location = location;
if (params != null) {
alter.params = new HashMap<>(params);
}
alter.purge = purge;
alter.ifNotExists = ifNotExists;
alter.ifExists = ifExists;
return alter;
}
private boolean processLine() {
log.debug("processing File:line " + Utils.getFileName(fileName) + this.lineNumber);
bins = new ArrayList<Bin>();
boolean lineProcessed = false;
long errorTotal = 0;
try {
if (columns.size() != counters.write.colTotal) {
if (columns.size() < counters.write.colTotal) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Number of column mismatch:Columns in data file is less than number of column in config file.");
} else {
throw new ParseException(lineNumber);
}
}
// retrieve set name first
for (ColumnDefinition metadataColumn : this.metadataMapping) {
if (metadataColumn.staticValue
&& metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.SET)) {
this.set = metadataColumn.binValueHeader;
} else {
String metadataRawText = this.columns.get(metadataColumn.getBinValuePos());
if (metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.SET)) {
if (this.set == null) {
this.set = metadataRawText;
}
}
}
}
// use set name to create key
for (ColumnDefinition metadataColumn : this.metadataMapping) {
if (metadataColumn.getBinNameHeader().equalsIgnoreCase(Constants.KEY)) {
String metadataRawText = this.columns.get(metadataColumn.getBinValuePos());
if (metadataColumn.getSrcType() == SrcColumnType.INTEGER) {
Long integer = Long.parseLong(metadataRawText);
this.key = new Key(this.nameSpace, this.set, integer);
} else {
this.key = new Key(this.nameSpace, this.set, metadataRawText);
}
}
}
for (ColumnDefinition binColumn : this.binMapping) {
Bin bin = null;
if (!binColumn.staticName) {
binColumn.binNameHeader = this.columns.get(binColumn.binNamePos);
}
if (!binColumn.staticValue) {
String binRawText = null;
if (binColumn.binValueHeader != null
&& binColumn.binValueHeader.toLowerCase().equals(Constants.SYSTEM_TIME)) {
SimpleDateFormat sdf = new SimpleDateFormat(binColumn.getEncoding()); // dd/MM/yyyy
Date now = new Date();
binRawText = sdf.format(now);
} else {
binRawText = this.columns.get(binColumn.getBinValuePos());
}
if (binRawText.equals("")) continue;
switch (binColumn.getSrcType()) {
case INTEGER:
// Server stores all integer type data in 64bit so use long
Long integer;
try {
integer = Long.parseLong(binRawText);
bin = new Bin(binColumn.getBinNameHeader(), integer);
} catch (Exception pi) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Integer/Long Parse Error:"
+ pi);
}
break;
case FLOAT:
/** Floating type data can be stored as 8 byte byte array. */
try {
float binfloat = Float.parseFloat(binRawText);
byte[] byteFloat = ByteBuffer.allocate(8).putFloat(binfloat).array();
bin = new Bin(binColumn.getBinNameHeader(), byteFloat);
} catch (Exception e) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Floating number Parse Error:"
+ e);
}
break;
case STRING:
bin = new Bin(binColumn.getBinNameHeader(), binRawText);
break;
case BLOB:
if (binColumn.getDstType().equals(DstColumnType.BLOB)) {
if (binColumn.encoding.equalsIgnoreCase(Constants.HEX_ENCODING))
bin = new Bin(binColumn.getBinNameHeader(), this.toByteArray(binRawText)); // TODO
}
break;
case LIST:
/*
* Assumptions
* 1. Items are separated by a colon ','
* 2. Item value will be a string
* 3. List will be in double quotes
*
* No support for nested maps or nested lists
*
*/
List<String> list = new ArrayList<String>();
String[] listValues = binRawText.split(Constants.LIST_DELEMITER, -1);
if (listValues.length > 0) {
for (String value : listValues) {
list.add(value.trim());
}
bin = Bin.asList(binColumn.getBinNameHeader(), list);
} else {
bin = null;
log.error("Error: Cannot parse to a list: " + binRawText);
}
break;
case MAP:
/*
* Asumptions:
* 1. Items are separated by a colon ','
* 2. Name value pairs are separated by equals ':'
* 3. Map key is a string
* 4. Map value will be a string
* 5. Map will be in double quotes
*
* No support for nested maps or nested lists
*
*/
Map<String, Object> map = new HashMap<String, Object>();
String[] mapValues = binRawText.split(Constants.MAP_DELEMITER, -1);
if (mapValues.length > 0) {
for (String value : mapValues) {
String[] kv = value.split(Constants.MAPKEY_DELEMITER);
if (kv.length != 2) log.error("Error: Cannot parse map <k,v> using: " + kv);
else map.put(kv[0].trim(), kv[1].trim());
}
log.debug(map.toString());
bin = Bin.asMap(binColumn.getBinNameHeader(), map);
} else {
bin = null;
log.error("Error: Cannot parse to a map: " + binRawText);
}
break;
case JSON:
try {
log.debug(binRawText);
if (jsonParser == null) jsonParser = new JSONParser();
Object obj = jsonParser.parse(binRawText);
if (obj instanceof JSONArray) {
JSONArray jsonArray = (JSONArray) obj;
bin = Bin.asList(binColumn.getBinNameHeader(), jsonArray);
} else {
JSONObject jsonObj = (JSONObject) obj;
bin = Bin.asMap(binColumn.getBinNameHeader(), jsonObj);
}
} catch (ParseException e) {
log.error("Failed to parse JSON", e);
}
break;
case TIMESTAMP:
if (binColumn.getDstType().equals(DstColumnType.INTEGER)) {
DateFormat format = new SimpleDateFormat(binColumn.getEncoding());
try {
Date formatDate = format.parse(binRawText);
long miliSecondForDate = formatDate.getTime() - timeZoneOffset;
if (binColumn.getEncoding().contains(".SSS")
&& binColumn.binValueHeader.toLowerCase().equals(Constants.SYSTEM_TIME)) {
// We need time in miliseconds so no need to change it to seconds
} else {
miliSecondForDate = miliSecondForDate / 1000;
}
bin = new Bin(binColumn.getBinNameHeader(), miliSecondForDate);
log.trace("Date format:" + binRawText + " in seconds:" + miliSecondForDate);
} catch (java.text.ParseException e) {
e.printStackTrace();
}
} else if (binColumn.getDstType().equals(DstColumnType.STRING)) {
bin = new Bin(binColumn.getBinNameHeader(), binRawText);
}
break;
default:
}
} else {
bin = new Bin(binColumn.getBinNameHeader(), binColumn.getBinValueHeader());
}
if (bin != null) {
bins.add(bin);
}
}
lineProcessed = true;
log.trace(
"Formed key and bins for line "
+ lineNumber
+ " Key: "
+ this.key.userKey
+ " Bins:"
+ this.bins.toString());
} catch (AerospikeException ae) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Aerospike Bin processing Error:"
+ ae.getResultCode());
if (log.isDebugEnabled()) {
ae.printStackTrace();
}
counters.write.processingErrors.getAndIncrement();
counters.write.recordProcessed.addAndGet(this.lineSize);
errorTotal =
(counters.write.readErrors.get()
+ counters.write.writeErrors.get()
+ counters.write.processingErrors.get());
if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) {
System.exit(-1);
}
} catch (ParseException pe) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Parsing Error:"
+ pe);
if (log.isDebugEnabled()) {
pe.printStackTrace();
}
counters.write.processingErrors.getAndIncrement();
counters.write.recordProcessed.addAndGet(this.lineSize);
errorTotal =
(counters.write.readErrors.get()
+ counters.write.writeErrors.get()
+ counters.write.processingErrors.get());
if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) {
System.exit(-1);
}
} catch (Exception e) {
log.error(
"File:"
+ Utils.getFileName(this.fileName)
+ " Line:"
+ lineNumber
+ " Unknown Error:"
+ e);
if (log.isDebugEnabled()) {
e.printStackTrace();
}
counters.write.processingErrors.getAndIncrement();
counters.write.recordProcessed.addAndGet(this.lineSize);
errorTotal =
(counters.write.readErrors.get()
+ counters.write.writeErrors.get()
+ counters.write.processingErrors.get());
if (this.abortErrorCount != 0 && this.abortErrorCount < errorTotal) {
System.exit(-1);
}
}
return lineProcessed;
}
