@SuppressWarnings("unchecked")
public <K, V> Map<K, V> getMap(int i, Class<K> keysClass, Class<V> valuesClass) {
DataType type = metadata.getType(i);
if (type.getName() != DataType.Name.MAP)
throw new InvalidTypeException(
String.format("Column %s is not of map type", metadata.getName(i)));
Class<?> expectedKeysClass = type.getTypeArguments().get(0).getName().javaType;
Class<?> expectedValuesClass = type.getTypeArguments().get(1).getName().javaType;
if (!keysClass.isAssignableFrom(expectedKeysClass)
|| !valuesClass.isAssignableFrom(expectedValuesClass))
throw new InvalidTypeException(
String.format(
"Column %s is a map of %s->%s (CQL type %s), cannot be retrieve as a map of %s->%s",
metadata.getName(i),
expectedKeysClass,
expectedValuesClass,
type,
keysClass,
valuesClass));
ByteBuffer value = data.get(i);
if (value == null) return Collections.<K, V>emptyMap();
return Collections.unmodifiableMap(Codec.<Map<K, V>>getCodec(type).compose(value));
}
DataType.Name checkType(int i, DataType.Name name1, DataType.Name name2, DataType.Name name3) {
DataType defined = getType(i);
if (name1 != defined.getName() && name2 != defined.getName() && name3 != defined.getName())
throw new InvalidTypeException(String.format("Column %s is of type %s", getName(i), defined));
return defined.getName();
}
/** Prints the sample collections that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
@SuppressWarnings("unchecked")
public void printSampleCollections() {
String objective = "Sample Collections";
System.out.println(String.format("Printing %s...", objective));
for (DataType dataType : SAMPLE_COLLECTIONS.keySet()) {
HashMap<DataType, Object> sampleValueMap =
(HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
if (dataType.getName() == DataType.Name.MAP) {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
HashMap<DataType, Object> sampleMap =
(HashMap<DataType, Object>) sampleValueMap.get(typeArgument);
Object mapKey = SAMPLE_DATA.get(typeArgument);
Object mapValue = sampleMap.get(typeArgument);
System.out.println(String.format("%1$-30s {%2$s : %3$s}", dataType, mapKey, mapValue));
} else {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
Object sampleValue = sampleValueMap.get(typeArgument);
System.out.println(String.format("%1$-30s %2$s", dataType, sampleValue));
}
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Generates the sample collections that will be used in testing */
private static HashMap<DataType, Object> getSampleCollections() {
HashMap<DataType, Object> sampleCollections = new HashMap<DataType, Object>();
HashMap<DataType, Object> setAndListCollection;
HashMap<DataType, HashMap<DataType, Object>> mapCollection;
for (DataType.Name dataTypeName : DATA_TYPE_NON_PRIMITIVE_NAMES) {
switch (dataTypeName) {
case LIST:
for (DataType typeArgument : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument)) continue;
List<Object> list = new ArrayList<Object>();
for (int i = 0; i < 5; i++) {
list.add(SAMPLE_DATA.get(typeArgument));
}
setAndListCollection = new HashMap<DataType, Object>();
setAndListCollection.put(typeArgument, list);
sampleCollections.put(DataType.list(typeArgument), setAndListCollection);
}
break;
case SET:
for (DataType typeArgument : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument)) continue;
Set<Object> set = new HashSet<Object>();
for (int i = 0; i < 5; i++) {
set.add(SAMPLE_DATA.get(typeArgument));
}
setAndListCollection = new HashMap<DataType, Object>();
setAndListCollection.put(typeArgument, set);
sampleCollections.put(DataType.set(typeArgument), setAndListCollection);
}
break;
case MAP:
for (DataType typeArgument1 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument1)) continue;
for (DataType typeArgument2 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument2)) continue;
HashMap<DataType, Object> map = new HashMap<DataType, Object>();
map.put(typeArgument1, SAMPLE_DATA.get(typeArgument2));
mapCollection = new HashMap<DataType, HashMap<DataType, Object>>();
mapCollection.put(typeArgument1, map);
sampleCollections.put(DataType.map(typeArgument1, typeArgument2), mapCollection);
}
}
break;
default:
throw new RuntimeException("Missing handling of " + dataTypeName);
}
}
return sampleCollections;
}
public static void toString(Object value, Text t) {
if (value == null) {
t.append("null");
} else {
final Class<? extends Object> clazz = value.getClass();
final DataType type = getDataType(clazz);
if (type == null) {
throw new RuntimeException("Invalid type: " + clazz.getName());
}
type.toMarkup(value, t);
}
}
/** Helper method to stringify SAMPLE_DATA for simple insert statements */
private static String helperStringifiedData(DataType dataType) {
String value = SAMPLE_DATA.get(dataType).toString();
switch (dataType.getName()) {
case BLOB:
value = "0xCAFE";
break;
case INET:
InetAddress v1 = (InetAddress) SAMPLE_DATA.get(dataType);
value = String.format("'%s'", v1.getHostAddress());
break;
case TIMESTAMP:
Date v2 = (Date) SAMPLE_DATA.get(dataType);
value = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(v2);
case ASCII:
case TEXT:
case VARCHAR:
value = String.format("'%s'", value);
break;
default:
break;
}
return value;
}
/**
* Given this type and another, find the 'broadest' type
*
* @param rhs The other type.
* @return The broadest type
* @throws TypeMismatchException Data does not match expected data type.
*/
public DataType getCommnType(DataType rhs) throws TypeMismatchException {
final DataType result;
result = ordinal() > rhs.ordinal() ? this : rhs;
if (numeric || rhs.numeric) {
// One numeric then both must be..
if (result.ordinal() > F.ordinal()) {
throw new TypeMismatchException("Either both must be numeric or neither");
}
} else {
// Neither numeric then both must be the same
if (ordinal() != rhs.ordinal()) {
throw new TypeMismatchException("Both types must be the same for non-numerics");
}
}
return result;
}
/**
* Given the parsing context and what numerical data type is expected convert a string to the
* correct type. Note no attempt is made to let the magnitude of the number influence our choice.
*
* @param string The string to convert to a number. E.g. "123.3e2". If it contains a '.' or an 'e'
* then the type must either be f or F.
* @param requiredType Either z, Z, f, F or any.
* @param tb The source. The cursor will be at the end of the number but any type specifier will
* not have been consumed. If there is one then we'll eat it.
* @return The derived type.
* @throws ParsingException If there is a clash of types.
*/
private static Object deriveType(String string, DataType requiredType, Text t, int radix) {
final Object result;
// Figure out the correct type...
final DataType derivedType;
if (t.isEof()) {
if (requiredType == DataType.any) {
if (string.indexOf('.') >= 0 || string.indexOf('e') >= 0) {
derivedType = DataType.f;
} else {
derivedType = DataType.z;
}
} else {
derivedType = requiredType;
}
} else {
final char c = t.peek();
if (c == 'z' || c == 'Z' || c == 'f' || c == 'F') {
t.consume(c);
derivedType = DataType.valueOf(String.valueOf(c));
if (!(requiredType == DataType.any || requiredType == derivedType)) {
throw new ParsingException("Incompatible type: " + string + c);
}
} else {
if (requiredType == DataType.any) {
if (string.indexOf('.') >= 0 || string.indexOf('e') >= 0) {
derivedType = DataType.f;
} else {
derivedType = DataType.z;
}
} else {
derivedType = requiredType;
}
}
}
switch (derivedType) {
case z:
result = new Long(Long.parseLong(string, radix));
break;
case Z:
result = new BigInteger(string, radix);
break;
case f:
result = new Double(string);
break;
case F:
result = new BigDecimal(string);
break;
// $CASES-OMITTED$
default:
throw new UnexpectedException("toType: " + derivedType);
}
return result;
}
@Override
public final boolean equals(Object o) {
if (!(o instanceof Definition)) return false;
Definition other = (Definition) o;
return keyspace.equals(other.keyspace)
&& table.equals(other.table)
&& name.equals(other.name)
&& type.equals(other.type);
}
@SuppressWarnings("unchecked")
public <T> Set<T> getSet(int i, Class<T> elementsClass) {
DataType type = metadata.getType(i);
if (type.getName() != DataType.Name.SET)
throw new InvalidTypeException(
String.format("Column %s is not of set type", metadata.getName(i)));
Class<?> expectedClass = type.getTypeArguments().get(0).getName().javaType;
if (!elementsClass.isAssignableFrom(expectedClass))
throw new InvalidTypeException(
String.format(
"Column %s is a set of %s (CQL type %s), cannot be retrieve as a set of %s",
metadata.getName(i), expectedClass, type, elementsClass));
ByteBuffer value = data.get(i);
if (value == null) return Collections.<T>emptySet();
return Collections.unmodifiableSet(Codec.<Set<T>>getCodec(type).compose(value));
}
/** Generates the insert statements that will be used in testing */
@SuppressWarnings("unchecked")
private static Collection<String> getCollectionInsertStatements() {
ArrayList<String> insertStatements = new ArrayList<String>();
String tableName;
String key;
String value;
for (DataType dataType : SAMPLE_COLLECTIONS.keySet()) {
HashMap<DataType, Object> sampleValueMap =
(HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
// Create tableName in form of: DataType_TypeArgument[_TypeArgument]
tableName = helperGenerateTableName(dataType);
if (dataType.getName() == DataType.Name.MAP) {
List<DataType> typeArgument = dataType.getTypeArguments();
key = helperStringifiedData(typeArgument.get(0));
value = helperStringifiedData(typeArgument.get(1));
insertStatements.add(String.format(MAP_INSERT_FORMAT, tableName, key, value));
} else if (dataType.getName() == DataType.Name.LIST) {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
key = helperStringifiedData(typeArgument);
// Create the value to be a list of the same 5 elements
value = "[";
for (int i = 0; i < 5; i++) value += key + ',';
value = value.substring(0, value.length() - 1) + ']';
insertStatements.add(String.format(COLLECTION_INSERT_FORMAT, tableName, key, value));
} else {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
key = helperStringifiedData(typeArgument);
value = '{' + key + '}';
insertStatements.add(String.format(COLLECTION_INSERT_FORMAT, tableName, key, value));
}
}
return insertStatements;
}
@SuppressWarnings("unchecked")
public <T> List<T> getList(int i, Class<T> elementsClass) {
DataType type = metadata.getType(i);
if (type.getName() != DataType.Name.LIST)
throw new InvalidTypeException(
String.format("Column %s is not of list type", metadata.getName(i)));
Class<?> expectedClass = type.getTypeArguments().get(0).getName().javaType;
if (!elementsClass.isAssignableFrom(expectedClass))
throw new InvalidTypeException(
String.format(
"Column %s is a list of %s (CQL type %s), cannot be retrieve as a list of %s",
metadata.getName(i), expectedClass, type, elementsClass));
ByteBuffer value = data.get(i);
if (value == null) return Collections.<T>emptyList();
// TODO: we could avoid the getCodec call if we kept a reference to the original message.
return Collections.unmodifiableList(Codec.<List<T>>getCodec(type).compose(value));
}
// The two following tests a really unit tests, but since the whole uses
// CCMBridge.PerClassSingleNodeCluster, they'll still spawn a cluster even
// you execute only them, so we keep them in the "long" group. We could
// move them in another class but not sure where honestly (one could argue
// that it would make more sense to move all the *other* tests to some
// DataTypeIntegrationTest class).
@Test(groups = "long")
public void serializeDeserializeTest() {
for (DataType dt : DataType.allPrimitiveTypes()) {
if (exclude(dt)) continue;
Object value = TestUtils.getFixedValue(dt);
assertEquals(dt.deserialize(dt.serialize(value)), value);
}
try {
DataType.bigint().serialize(4);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
try {
ByteBuffer badValue = ByteBuffer.allocate(4);
DataType.bigint().deserialize(badValue);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
}
/**
* Return the DataType corresponding to a given Java type. For example 'text' is returned given
* String.class.
*
* @param clazz The class to look up.
* @return The associated class or null if not found.
*/
@Nullable
public static DataType getDataType(Class<?> clazz) {
/*
* This has to be done just-in-time as the javaToDataType is static and
* can't be loaded as the enum is created.
*/
if (javaToDataType.isEmpty()) {
for (final DataType dataType : DataType.values()) {
javaToDataType.put(dataType.javaClass, dataType);
}
}
return javaToDataType.get(clazz);
}
/** Test simple statement selects for all collection data types */
@SuppressWarnings("unchecked")
public void collectionSelectTest() throws Throwable {
HashMap<DataType, Object> sampleValueMap;
String execute_string;
DataType typeArgument1;
DataType typeArgument2;
Row row;
for (DataType dataType : COLLECTION_SELECT_STATEMENTS.keySet()) {
execute_string = COLLECTION_SELECT_STATEMENTS.get(dataType);
row = session.execute(execute_string).one();
sampleValueMap = (HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
typeArgument1 = dataType.getTypeArguments().get(0);
if (dataType.getName() == DataType.Name.MAP) {
typeArgument2 = dataType.getTypeArguments().get(1);
// Create a copy of the map that is being expected
HashMap<DataType, Object> sampleMap =
(HashMap<DataType, Object>) sampleValueMap.get(typeArgument1);
Object mapKey = SAMPLE_DATA.get(sampleMap.keySet().iterator().next());
Object mapValue = sampleMap.values().iterator().next();
HashMap<Object, Object> expectedMap = new HashMap<Object, Object>();
expectedMap.put(mapKey, mapValue);
assertEquals(TestUtils.getValue(row, "k", typeArgument2), SAMPLE_DATA.get(typeArgument2));
assertEquals(TestUtils.getValue(row, "v", dataType), expectedMap);
} else {
Object expectedValue = sampleValueMap.get(typeArgument1);
assertEquals(TestUtils.getValue(row, "k", typeArgument1), SAMPLE_DATA.get(typeArgument1));
assertEquals(TestUtils.getValue(row, "v", dataType), expectedValue);
}
}
assertEquals(SAMPLE_COLLECTIONS.size(), 255);
assertEquals(COLLECTION_SELECT_STATEMENTS.keySet().size(), SAMPLE_COLLECTIONS.size());
}
// This method checks the value of the "[applied]" column manually, to avoid instantiating an
// ArrayBackedRow
// object that we would throw away immediately.
private static boolean checkWasApplied(List<ByteBuffer> firstRow, ColumnDefinitions metadata) {
// If the column is not present or not a boolean, we assume the query
// was not a conditional statement, and therefore return true.
if (firstRow == null) return true;
int[] is = metadata.findAllIdx("[applied]");
if (is == null) return true;
int i = is[0];
if (!DataType.cboolean().equals(metadata.getType(i))) return true;
// Otherwise return the value of the column
ByteBuffer value = firstRow.get(i);
if (value == null || value.remaining() == 0) return false;
return TypeCodec.BooleanCodec.instance.deserializeNoBoxing(value);
}
@Test(groups = "long")
public void serializeDeserializeCollectionsTest() {
List<String> l = Arrays.asList("foo", "bar");
DataType dt = DataType.list(DataType.text());
assertEquals(dt.deserialize(dt.serialize(l)), l);
try {
DataType.list(DataType.bigint()).serialize(l);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
}
/**
* Can't override valueOf() so uUse this method instead as it treats boolean properly. If an error
* is detected the Advisory is updated.
*
* @param key The key to look up.
* @return The DataType associated with the given key.
*/
public static DataType valueOfCorrected(String key) {
DataType result;
try {
if ("boolean".equals(key)) {
result = bool;
} else {
result = DataType.valueOf(key);
}
} catch (final IllegalArgumentException e) {
// todo Perhaps throw the catch when the Locus is known?
final Advisory advisory = Context.get(Advisory.class);
advisory.error("Invalid data type: " + key);
result = any;
}
return result;
}
public void setType(String type) {
this.type = DataType.valueOf(type.trim().toUpperCase());
}
/** Helper method to generate table names in the form of: DataType_TypeArgument[_TypeArgument] */
private static String helperGenerateTableName(DataType dataType) {
String tableName = dataType.getName().toString();
for (DataType typeArgument : dataType.getTypeArguments()) tableName += "_" + typeArgument;
return tableName;
}
/**
* Tests DataType class to ensure data sent in is the same as data received All tests are executed
* via a Simple Statements Counters are the only datatype not tested within the entirety of the
* suite. There is, however, an isolated test case that needs to be implemented. All statements and
* sample data is easily exportable via the print_*() methods.
*/
public class DataTypeTest extends CCMBridge.PerClassSingleNodeCluster {
private static final Set<DataType> DATA_TYPE_PRIMITIVES = DataType.allPrimitiveTypes();
private static final Set<DataType.Name> DATA_TYPE_NON_PRIMITIVE_NAMES =
EnumSet.of(DataType.Name.MAP, DataType.Name.SET, DataType.Name.LIST);
private static final String PRIMITIVE_INSERT_FORMAT =
"INSERT INTO %1$s (k, v) VALUES (%2$s, %2$s);";
private static final String BASIC_SELECT_FORMAT = "SELECT k, v FROM %1$s;";
private static final String COLLECTION_INSERT_FORMAT =
"INSERT INTO %1$s (k, v) VALUES (%2$s, %3$s);";
private static final String MAP_INSERT_FORMAT =
"INSERT INTO %1$s (k, v) VALUES (%3$s, {%2$s: %3$s});";
private static final HashMap<DataType, Object> SAMPLE_DATA = getSampleData();
private static final HashMap<DataType, Object> SAMPLE_COLLECTIONS = getSampleCollections();
private static final Collection<String> PRIMITIVE_INSERT_STATEMENTS =
getPrimitiveInsertStatements();
private static final HashMap<DataType, String> PRIMITIVE_SELECT_STATEMENTS =
getPrimitiveSelectStatements();
private static final Collection<String> COLLECTION_INSERT_STATEMENTS =
getCollectionInsertStatements();
private static final HashMap<DataType, String> COLLECTION_SELECT_STATEMENTS =
getCollectionSelectStatements();
private static boolean exclude(DataType t) {
return t.getName() == DataType.Name.COUNTER;
}
/** Generates the table definitions that will be used in testing */
@Override
protected Collection<String> getTableDefinitions() {
ArrayList<String> tableDefinitions = new ArrayList<String>();
// Create primitive data type definitions
for (DataType dataType : DATA_TYPE_PRIMITIVES) {
if (exclude(dataType)) continue;
tableDefinitions.add(
String.format("CREATE TABLE %1$s (k %2$s PRIMARY KEY, v %1$s)", dataType, dataType));
}
// Create collection data type definitions
for (DataType.Name dataTypeName : DATA_TYPE_NON_PRIMITIVE_NAMES) {
// Create MAP data type definitions
if (dataTypeName == DataType.Name.MAP) {
for (DataType typeArgument1 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument1)) continue;
for (DataType typeArgument2 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument2)) continue;
tableDefinitions.add(
String.format(
"CREATE TABLE %1$s_%2$s_%3$s (k %3$s PRIMARY KEY, v %1$s<%2$s, %3$s>)",
dataTypeName, typeArgument1, typeArgument2));
}
}
// Create SET and LIST data type definitions
} else {
for (DataType typeArgument : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument)) continue;
tableDefinitions.add(
String.format(
"CREATE TABLE %1$s_%2$s (k %2$s PRIMARY KEY, v %1$s<%2$s>)",
dataTypeName, typeArgument));
}
}
}
return tableDefinitions;
}
/** Generates the sample data that will be used in testing */
private static HashMap<DataType, Object> getSampleData() {
HashMap<DataType, Object> sampleData = new HashMap<DataType, Object>();
for (DataType dataType : DATA_TYPE_PRIMITIVES) {
switch (dataType.getName()) {
case ASCII:
sampleData.put(dataType, new String("ascii"));
break;
case BIGINT:
sampleData.put(dataType, Long.MAX_VALUE);
break;
case BLOB:
ByteBuffer bb = ByteBuffer.allocate(58);
bb.putShort((short) 0xCAFE);
bb.flip();
sampleData.put(dataType, bb);
break;
case BOOLEAN:
sampleData.put(dataType, Boolean.TRUE);
break;
case COUNTER:
// Not supported in an insert statement
break;
case DECIMAL:
sampleData.put(dataType, new BigDecimal("12.3E+7"));
break;
case DOUBLE:
sampleData.put(dataType, Double.MAX_VALUE);
break;
case FLOAT:
sampleData.put(dataType, Float.MAX_VALUE);
break;
case INET:
try {
sampleData.put(dataType, InetAddress.getByName("123.123.123.123"));
} catch (java.net.UnknownHostException e) {
}
break;
case INT:
sampleData.put(dataType, Integer.MAX_VALUE);
break;
case TEXT:
sampleData.put(dataType, new String("text"));
break;
case TIMESTAMP:
sampleData.put(dataType, new Date(872835240000L));
break;
case TIMEUUID:
sampleData.put(dataType, UUID.fromString("FE2B4360-28C6-11E2-81C1-0800200C9A66"));
break;
case UUID:
sampleData.put(dataType, UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff00"));
break;
case VARCHAR:
sampleData.put(dataType, new String("varchar"));
break;
case VARINT:
sampleData.put(dataType, new BigInteger(Integer.toString(Integer.MAX_VALUE) + "000"));
break;
default:
throw new RuntimeException("Missing handling of " + dataType);
}
}
return sampleData;
}
/** Generates the sample collections that will be used in testing */
private static HashMap<DataType, Object> getSampleCollections() {
HashMap<DataType, Object> sampleCollections = new HashMap<DataType, Object>();
HashMap<DataType, Object> setAndListCollection;
HashMap<DataType, HashMap<DataType, Object>> mapCollection;
for (DataType.Name dataTypeName : DATA_TYPE_NON_PRIMITIVE_NAMES) {
switch (dataTypeName) {
case LIST:
for (DataType typeArgument : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument)) continue;
List<Object> list = new ArrayList<Object>();
for (int i = 0; i < 5; i++) {
list.add(SAMPLE_DATA.get(typeArgument));
}
setAndListCollection = new HashMap<DataType, Object>();
setAndListCollection.put(typeArgument, list);
sampleCollections.put(DataType.list(typeArgument), setAndListCollection);
}
break;
case SET:
for (DataType typeArgument : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument)) continue;
Set<Object> set = new HashSet<Object>();
for (int i = 0; i < 5; i++) {
set.add(SAMPLE_DATA.get(typeArgument));
}
setAndListCollection = new HashMap<DataType, Object>();
setAndListCollection.put(typeArgument, set);
sampleCollections.put(DataType.set(typeArgument), setAndListCollection);
}
break;
case MAP:
for (DataType typeArgument1 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument1)) continue;
for (DataType typeArgument2 : DATA_TYPE_PRIMITIVES) {
if (exclude(typeArgument2)) continue;
HashMap<DataType, Object> map = new HashMap<DataType, Object>();
map.put(typeArgument1, SAMPLE_DATA.get(typeArgument2));
mapCollection = new HashMap<DataType, HashMap<DataType, Object>>();
mapCollection.put(typeArgument1, map);
sampleCollections.put(DataType.map(typeArgument1, typeArgument2), mapCollection);
}
}
break;
default:
throw new RuntimeException("Missing handling of " + dataTypeName);
}
}
return sampleCollections;
}
/** Helper method to stringify SAMPLE_DATA for simple insert statements */
private static String helperStringifiedData(DataType dataType) {
String value = SAMPLE_DATA.get(dataType).toString();
switch (dataType.getName()) {
case BLOB:
value = "0xCAFE";
break;
case INET:
InetAddress v1 = (InetAddress) SAMPLE_DATA.get(dataType);
value = String.format("'%s'", v1.getHostAddress());
break;
case TIMESTAMP:
Date v2 = (Date) SAMPLE_DATA.get(dataType);
value = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(v2);
case ASCII:
case TEXT:
case VARCHAR:
value = String.format("'%s'", value);
break;
default:
break;
}
return value;
}
/** Generates the insert statements that will be used in testing */
private static Collection<String> getPrimitiveInsertStatements() {
ArrayList<String> insertStatements = new ArrayList<String>();
for (DataType dataType : SAMPLE_DATA.keySet()) {
String value = helperStringifiedData(dataType);
insertStatements.add(String.format(PRIMITIVE_INSERT_FORMAT, dataType, value));
}
return insertStatements;
}
/** Generates the select statements that will be used in testing */
private static HashMap<DataType, String> getPrimitiveSelectStatements() {
HashMap<DataType, String> selectStatements = new HashMap<DataType, String>();
for (DataType dataType : SAMPLE_DATA.keySet()) {
selectStatements.put(dataType, String.format(BASIC_SELECT_FORMAT, dataType));
}
return selectStatements;
}
/** Helper method to generate table names in the form of: DataType_TypeArgument[_TypeArgument] */
private static String helperGenerateTableName(DataType dataType) {
String tableName = dataType.getName().toString();
for (DataType typeArgument : dataType.getTypeArguments()) tableName += "_" + typeArgument;
return tableName;
}
/** Generates the insert statements that will be used in testing */
@SuppressWarnings("unchecked")
private static Collection<String> getCollectionInsertStatements() {
ArrayList<String> insertStatements = new ArrayList<String>();
String tableName;
String key;
String value;
for (DataType dataType : SAMPLE_COLLECTIONS.keySet()) {
HashMap<DataType, Object> sampleValueMap =
(HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
// Create tableName in form of: DataType_TypeArgument[_TypeArgument]
tableName = helperGenerateTableName(dataType);
if (dataType.getName() == DataType.Name.MAP) {
List<DataType> typeArgument = dataType.getTypeArguments();
key = helperStringifiedData(typeArgument.get(0));
value = helperStringifiedData(typeArgument.get(1));
insertStatements.add(String.format(MAP_INSERT_FORMAT, tableName, key, value));
} else if (dataType.getName() == DataType.Name.LIST) {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
key = helperStringifiedData(typeArgument);
// Create the value to be a list of the same 5 elements
value = "[";
for (int i = 0; i < 5; i++) value += key + ',';
value = value.substring(0, value.length() - 1) + ']';
insertStatements.add(String.format(COLLECTION_INSERT_FORMAT, tableName, key, value));
} else {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
key = helperStringifiedData(typeArgument);
value = '{' + key + '}';
insertStatements.add(String.format(COLLECTION_INSERT_FORMAT, tableName, key, value));
}
}
return insertStatements;
}
/** Generates the select statements that will be used in testing */
private static HashMap<DataType, String> getCollectionSelectStatements() {
HashMap<DataType, String> selectStatements = new HashMap<DataType, String>();
String tableName;
for (DataType dataType : SAMPLE_COLLECTIONS.keySet()) {
tableName = helperGenerateTableName(dataType);
selectStatements.put(dataType, String.format(BASIC_SELECT_FORMAT, tableName));
}
return selectStatements;
}
/** Test simple statement inserts for all primitive data types */
public void primitiveInsertTest() throws Throwable {
ResultSet rs;
for (String execute_string : PRIMITIVE_INSERT_STATEMENTS) {
rs = session.execute(execute_string);
assertTrue(rs.isExhausted());
}
assertEquals(SAMPLE_DATA.size(), 15);
assertEquals(PRIMITIVE_INSERT_STATEMENTS.size(), SAMPLE_DATA.size());
}
/** Validate simple statement selects for all primitive data types */
public void primitiveSelectTest() throws Throwable {
String execute_string;
Object value;
Row row;
for (DataType dataType : PRIMITIVE_SELECT_STATEMENTS.keySet()) {
execute_string = PRIMITIVE_SELECT_STATEMENTS.get(dataType);
row = session.execute(execute_string).one();
value = SAMPLE_DATA.get(dataType);
assertEquals(TestUtils.getValue(row, "k", dataType), value);
assertEquals(TestUtils.getValue(row, "v", dataType), value);
}
assertEquals(SAMPLE_DATA.size(), 15);
assertEquals(PRIMITIVE_SELECT_STATEMENTS.keySet().size(), SAMPLE_DATA.size());
}
/** Test simple statement inserts and selects for all primitive data types */
@Test(groups = "long")
public void primitiveTests() throws Throwable {
primitiveInsertTest();
primitiveSelectTest();
}
/** Test simple statement inserts for all collection data types */
public void collectionInsertTest() throws Throwable {
ResultSet rs;
for (String execute_string : COLLECTION_INSERT_STATEMENTS) {
rs = session.execute(execute_string);
assertTrue(rs.isExhausted());
}
assertEquals(SAMPLE_COLLECTIONS.size(), 255);
assertEquals(COLLECTION_INSERT_STATEMENTS.size(), SAMPLE_COLLECTIONS.size());
}
/** Test simple statement selects for all collection data types */
@SuppressWarnings("unchecked")
public void collectionSelectTest() throws Throwable {
HashMap<DataType, Object> sampleValueMap;
String execute_string;
DataType typeArgument1;
DataType typeArgument2;
Row row;
for (DataType dataType : COLLECTION_SELECT_STATEMENTS.keySet()) {
execute_string = COLLECTION_SELECT_STATEMENTS.get(dataType);
row = session.execute(execute_string).one();
sampleValueMap = (HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
typeArgument1 = dataType.getTypeArguments().get(0);
if (dataType.getName() == DataType.Name.MAP) {
typeArgument2 = dataType.getTypeArguments().get(1);
// Create a copy of the map that is being expected
HashMap<DataType, Object> sampleMap =
(HashMap<DataType, Object>) sampleValueMap.get(typeArgument1);
Object mapKey = SAMPLE_DATA.get(sampleMap.keySet().iterator().next());
Object mapValue = sampleMap.values().iterator().next();
HashMap<Object, Object> expectedMap = new HashMap<Object, Object>();
expectedMap.put(mapKey, mapValue);
assertEquals(TestUtils.getValue(row, "k", typeArgument2), SAMPLE_DATA.get(typeArgument2));
assertEquals(TestUtils.getValue(row, "v", dataType), expectedMap);
} else {
Object expectedValue = sampleValueMap.get(typeArgument1);
assertEquals(TestUtils.getValue(row, "k", typeArgument1), SAMPLE_DATA.get(typeArgument1));
assertEquals(TestUtils.getValue(row, "v", dataType), expectedValue);
}
}
assertEquals(SAMPLE_COLLECTIONS.size(), 255);
assertEquals(COLLECTION_SELECT_STATEMENTS.keySet().size(), SAMPLE_COLLECTIONS.size());
}
/** Test simple statement inserts and selects for all collection data types */
@Test(groups = "long")
public void collectionTest() throws Throwable {
collectionInsertTest();
collectionSelectTest();
}
// The two following tests a really unit tests, but since the whole uses
// CCMBridge.PerClassSingleNodeCluster, they'll still spawn a cluster even
// you execute only them, so we keep them in the "long" group. We could
// move them in another class but not sure where honestly (one could argue
// that it would make more sense to move all the *other* tests to some
// DataTypeIntegrationTest class).
@Test(groups = "long")
public void serializeDeserializeTest() {
for (DataType dt : DataType.allPrimitiveTypes()) {
if (exclude(dt)) continue;
Object value = TestUtils.getFixedValue(dt);
assertEquals(dt.deserialize(dt.serialize(value)), value);
}
try {
DataType.bigint().serialize(4);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
try {
ByteBuffer badValue = ByteBuffer.allocate(4);
DataType.bigint().deserialize(badValue);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
}
@Test(groups = "long")
public void serializeDeserializeCollectionsTest() {
List<String> l = Arrays.asList("foo", "bar");
DataType dt = DataType.list(DataType.text());
assertEquals(dt.deserialize(dt.serialize(l)), l);
try {
DataType.list(DataType.bigint()).serialize(l);
fail("This should not have worked");
} catch (InvalidTypeException e) {
/* That's what we want */
}
}
/** Prints the table definitions that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printTableDefinitions() {
String objective = "Table Definitions";
System.out.println(String.format("Printing %s...", objective));
// Prints the full list of table definitions
for (String definition : getTableDefinitions()) {
System.out.println(definition);
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the sample data that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printSampleData() {
String objective = "Sample Data";
System.out.println(String.format("Printing %s...", objective));
for (DataType dataType : SAMPLE_DATA.keySet()) {
Object sampleValue = SAMPLE_DATA.get(dataType);
System.out.println(String.format("%1$-10s %2$s", dataType, sampleValue));
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the sample collections that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
@SuppressWarnings("unchecked")
public void printSampleCollections() {
String objective = "Sample Collections";
System.out.println(String.format("Printing %s...", objective));
for (DataType dataType : SAMPLE_COLLECTIONS.keySet()) {
HashMap<DataType, Object> sampleValueMap =
(HashMap<DataType, Object>) SAMPLE_COLLECTIONS.get(dataType);
if (dataType.getName() == DataType.Name.MAP) {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
HashMap<DataType, Object> sampleMap =
(HashMap<DataType, Object>) sampleValueMap.get(typeArgument);
Object mapKey = SAMPLE_DATA.get(typeArgument);
Object mapValue = sampleMap.get(typeArgument);
System.out.println(String.format("%1$-30s {%2$s : %3$s}", dataType, mapKey, mapValue));
} else {
DataType typeArgument = sampleValueMap.keySet().iterator().next();
Object sampleValue = sampleValueMap.get(typeArgument);
System.out.println(String.format("%1$-30s %2$s", dataType, sampleValue));
}
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the simple insert statements that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printPrimitiveInsertStatements() {
String objective = "Primitive Insert Statements";
System.out.println(String.format("Printing %s...", objective));
for (String execute_string : PRIMITIVE_INSERT_STATEMENTS) {
System.out.println(execute_string);
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the simple select statements that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printPrimitiveSelectStatements() {
String objective = "Primitive Select Statements";
System.out.println(String.format("Printing %s...", objective));
for (String execute_string : PRIMITIVE_SELECT_STATEMENTS.values()) {
System.out.println(execute_string);
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the simple insert statements that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printCollectionInsertStatements() {
String objective = "Collection Insert Statements";
System.out.println(String.format("Printing %s...", objective));
for (String execute_string : COLLECTION_INSERT_STATEMENTS) {
System.out.println(execute_string);
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
/** Prints the simple insert statements that will be used in testing (for exporting purposes) */
@Test(groups = "doc")
public void printCollectionSelectStatements() {
String objective = "Collection Select Statements";
System.out.println(String.format("Printing %s...", objective));
for (String execute_string : COLLECTION_SELECT_STATEMENTS.values()) {
System.out.println(execute_string);
}
System.out.println(String.format("\nEnd of %s\n\n", objective));
}
}
// Note: we avoid having a vararg method to avoid the array allocation that comes with it.
void checkType(int i, DataType.Name name) {
DataType defined = getType(i);
if (name != defined.getName())
throw new InvalidTypeException(String.format("Column %s is of type %s", getName(i), defined));
}
private static boolean exclude(DataType t) {
return t.getName() == DataType.Name.COUNTER;
}
/** Generates the sample data that will be used in testing */
private static HashMap<DataType, Object> getSampleData() {
HashMap<DataType, Object> sampleData = new HashMap<DataType, Object>();
for (DataType dataType : DATA_TYPE_PRIMITIVES) {
switch (dataType.getName()) {
case ASCII:
sampleData.put(dataType, new String("ascii"));
break;
case BIGINT:
sampleData.put(dataType, Long.MAX_VALUE);
break;
case BLOB:
ByteBuffer bb = ByteBuffer.allocate(58);
bb.putShort((short) 0xCAFE);
bb.flip();
sampleData.put(dataType, bb);
break;
case BOOLEAN:
sampleData.put(dataType, Boolean.TRUE);
break;
case COUNTER:
// Not supported in an insert statement
break;
case DECIMAL:
sampleData.put(dataType, new BigDecimal("12.3E+7"));
break;
case DOUBLE:
sampleData.put(dataType, Double.MAX_VALUE);
break;
case FLOAT:
sampleData.put(dataType, Float.MAX_VALUE);
break;
case INET:
try {
sampleData.put(dataType, InetAddress.getByName("123.123.123.123"));
} catch (java.net.UnknownHostException e) {
}
break;
case INT:
sampleData.put(dataType, Integer.MAX_VALUE);
break;
case TEXT:
sampleData.put(dataType, new String("text"));
break;
case TIMESTAMP:
sampleData.put(dataType, new Date(872835240000L));
break;
case TIMEUUID:
sampleData.put(dataType, UUID.fromString("FE2B4360-28C6-11E2-81C1-0800200C9A66"));
break;
case UUID:
sampleData.put(dataType, UUID.fromString("067e6162-3b6f-4ae2-a171-2470b63dff00"));
break;
case VARCHAR:
sampleData.put(dataType, new String("varchar"));
break;
case VARINT:
sampleData.put(dataType, new BigInteger(Integer.toString(Integer.MAX_VALUE) + "000"));
break;
default:
throw new RuntimeException("Missing handling of " + dataType);
}
}
return sampleData;
}
@SuppressWarnings("unchecked")
@Override
public <P> Getter<GettableByIndexData, P> newGetter(
Type target, DatastaxColumnKey key, ColumnDefinition<?, ?> columnDefinition) {
Class<?> targetClass = TypeHelper.toClass(target);
if (Date.class.equals(targetClass)) {
return (Getter<GettableByIndexData, P>) new DatastaxDateGetter(key.getIndex());
}
if (boolean.class.equals(targetClass) || Boolean.class.equals(targetClass)) {
return (Getter<GettableByIndexData, P>) new DatastaxBooleanGetter(key.getIndex());
}
if (InetAddress.class.equals(targetClass)) {
return (Getter<GettableByIndexData, P>) new DatastaxInetAddressGetter(key.getIndex());
}
if (TupleValue.class.equals(targetClass)) {
return (Getter<GettableByIndexData, P>) new DatastaxTupleValueGetter(key.getIndex());
}
if (Collection.class.isAssignableFrom(targetClass)) {
Type elementType = TypeHelper.getComponentTypeOfListOrArray(target);
Class<?> dataTypeClass = Object.class;
Class<?> dataTypeElt = null;
DataType dtElt = null;
if (key.getDataType() != null) {
DataType dataType = key.getDataType();
dataTypeClass = dataType.asJavaClass();
if (dataType.isCollection()) {
dtElt = key.getDataType().getTypeArguments().get(0);
dataTypeElt = dtElt.asJavaClass();
}
} else {
dataTypeElt = TypeHelper.toClass(elementType);
}
if (dataTypeElt != null) {
if (TypeHelper.areEquals(elementType, dataTypeElt)) {
if (Set.class.equals(dataTypeClass)) {
if (targetClass.isAssignableFrom(dataTypeClass)) {
return new DatastaxSetGetter(key.getIndex(), TypeHelper.toClass(elementType));
}
}
if (List.class.equals(dataTypeClass)) {
if (targetClass.isAssignableFrom(dataTypeClass)) {
return new DatastaxListGetter(key.getIndex(), TypeHelper.toClass(elementType));
}
}
} else {
Converter<?, ?> converter = getConverter(elementType, dataTypeElt, dtElt);
if (converter != null) {
if (Set.class.equals(dataTypeClass)) {
if (targetClass.isAssignableFrom(dataTypeClass)) {
return new DatastaxSetWithConverterGetter(key.getIndex(), dataTypeElt, converter);
}
}
if (List.class.equals(dataTypeClass)) {
if (targetClass.isAssignableFrom(dataTypeClass)) {
return new DatastaxListWithConverterGetter(key.getIndex(), dataTypeElt, converter);
}
}
}
}
}
}
if (Map.class.equals(targetClass)) {
Tuple2<Type, Type> keyValueTypeOfMap = TypeHelper.getKeyValueTypeOfMap(target);
Class<?> dtKeyType = null;
Class<?> dtValueType = null;
DataType dtKey = null;
DataType dtValue = null;
if (key.getDataType() != null) {
List<DataType> typeArguments = key.getDataType().getTypeArguments();
if (typeArguments.size() == 2) {
dtKey = typeArguments.get(0);
dtKeyType = dtKey.asJavaClass();
dtValue = typeArguments.get(1);
dtValueType = dtValue.asJavaClass();
}
} else {
dtKeyType = TypeHelper.toClass(keyValueTypeOfMap.first());
dtValueType = TypeHelper.toClass(keyValueTypeOfMap.second());
}
if (dtKeyType != null && dtValueType != null) {
if (TypeHelper.areEquals(keyValueTypeOfMap.first(), dtKeyType)
&& TypeHelper.areEquals(keyValueTypeOfMap.second(), dtValueType)) {
return new DatastaxMapGetter(
key.getIndex(),
TypeHelper.toClass(keyValueTypeOfMap.first()),
TypeHelper.toClass(keyValueTypeOfMap.second()));
} else {
Converter<?, ?> keyConverter = getConverter(keyValueTypeOfMap.first(), dtKeyType, dtKey);
Converter<?, ?> valueConverter =
getConverter(keyValueTypeOfMap.second(), dtValueType, dtValue);
if (keyConverter != null && valueConverter != null) {
return new DatastaxMapWithConverterGetter(
key.getIndex(), dtKeyType, dtValueType, keyConverter, valueConverter);
}
}
}
}
if (Tuples.isTuple(target)) {
if (key.getDataType() != null && key.getDataType() instanceof TupleType) {
TupleType tt = (TupleType) key.getDataType();
List<DataType> typeArguments = tt.getTypeArguments();
TypeVariable<? extends Class<?>>[] typeParameters = targetClass.getTypeParameters();
if (typeArguments.size() <= typeParameters.length) {
return (Getter<GettableByIndexData, P>)
DatastaxTupleGetter.newInstance(datastaxMapperFactory, target, tt, key.getIndex());
}
}
}
if (TypeHelper.isEnum(target)) {
final Getter<GettableByIndexData, ? extends Enum> getter =
enumGetter(key, TypeHelper.toClass(target));
if (getter != null) {
return (Getter<GettableByIndexData, P>) getter;
}
}
final GetterFactory<GettableByIndexData, DatastaxColumnKey> rowGetterFactory =
getterFactories.get(targetClass);
if (rowGetterFactory != null) {
return rowGetterFactory.newGetter(target, key, columnDefinition);
}
final Getter<GettableByIndexData, P> getter =
jodaTimeGetterFactory.newGetter(target, key, columnDefinition);
if (getter != null) {
return getter;
}
if (key.getDataType() != null && key.getDataType() instanceof UserType) {
UserType ut = (UserType) key.getDataType();
return (Getter<GettableByIndexData, P>)
DatastaxUDTGetter.newInstance(datastaxMapperFactory, target, ut, key.getIndex());
}
return null;
}
@Override
protected BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec)
throws Exception {
List<File> inputFiles =
FileSelectPanel.getInputFiles(propInputDir.getStringValue(), getAllowedFileExtensions());
if (inputFiles.isEmpty()) {
throw new RuntimeException("No files selected");
}
// first group files into plate-groups
Map<String, List<File>> plateFiles = splitFilesIntoPlates(inputFiles);
if (inputFiles.isEmpty()) {
throw new RuntimeException("No valid envision-files in selection " + inputFiles);
}
// split files
List<String> allAttributes = mergeAttributes(plateFiles);
List<Attribute> colAttributes = compileColumnModel(allAttributes);
DataTableSpec outputSpec = AttributeUtils.compileTableSpecs(colAttributes);
BufferedDataContainer container = exec.createDataContainer(outputSpec);
// populate the table
int fileCounter = 0, rowCounter = 0;
for (String barcode : plateFiles.keySet()) {
logger.info("Processing plate " + barcode);
Plate plate = new Plate();
// invalidate plate-dims as these become fixed in the loop
plate.setNumColumns(-1);
plate.setNumRows(-1);
for (File file : plateFiles.get(barcode)) {
String attributeName = getAttributeNameOfEnvisionFile(file);
parseFile(plate, attributeName, file);
BufTableUtils.updateProgress(exec, fileCounter++, inputFiles.size());
}
// now create the data-rows for this table
for (Well well : plate.getWells()) {
if (well.getReadOutNames().isEmpty()) {
continue;
}
DataCell[] knimeRow = new DataCell[colAttributes.size()];
// first add the barcode-column
knimeRow[0] = new StringCell(barcode);
knimeRow[1] = colAttributes.get(1).createCell(well.getPlateRow());
knimeRow[2] = colAttributes.get(2).createCell(well.getPlateColumn());
for (String attributeName : allAttributes) {
int rowIndex = allAttributes.indexOf(attributeName);
Double value = well.getReadout(attributeName);
if (value != null) {
knimeRow[3 + rowIndex] = new DoubleCell(value);
} else {
knimeRow[3 + rowIndex] = DataType.getMissingCell();
}
}
DataRow tableRow = new DefaultRow(new RowKey("" + rowCounter++), knimeRow);
container.addRowToTable(tableRow);
}
}
container.close();
return new BufferedDataTable[] {container.getTable()};
}