/** Initialize dictionaries for dimension values */
private void initializeAndBuildDictionaries(
Schema schema, Map<String, ColumnIndexCreationInfo> columnInfo, File file) throws Exception {
for (final FieldSpec spec : schema.getAllFieldSpecs()) {
final ColumnIndexCreationInfo info = columnInfo.get(spec.getName());
if (info.isCreateDictionary()) {
dictionaryCreatorMap.put(
spec.getName(),
new SegmentDictionaryCreator(
info.hasNulls(), info.getSortedUniqueElementsArray(), spec, file));
} else {
throw new RuntimeException("Creation of indices without dictionaries is not implemented!");
}
dictionaryCreatorMap.get(spec.getName()).build();
}
// Add __ALL__ to dimension dictionaries
for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) {
Object allValue = StarTreeIndexNode.getAllValue(spec);
if (schema.getFieldSpecFor(spec.getName()).isSingleValueField()) {
Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfSV(allValue);
} else {
Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfMV(allValue);
}
}
}
public RelOptTableImpl getTable(final String[] names) {
List<Pair<String, Object>> pairs = new ArrayList<Pair<String, Object>>();
Schema schema2 = schema;
for (int i = 0; i < names.length; i++) {
final String name = names[i];
Schema subSchema = schema2.getSubSchema(name);
if (subSchema != null) {
pairs.add(Pair.<String, Object>of(name, subSchema));
schema2 = subSchema;
continue;
}
final Table table = schema2.getTable(name);
if (table != null) {
pairs.add(Pair.<String, Object>of(name, table));
if (i != names.length - 1) {
// not enough objects to match all names
return null;
}
return new RelOptTableImpl(
this, typeFactory.createType(table.getElementType()), names, table);
}
return null;
}
return null;
}
public final String getJdbcSchemaName(final Schema schema) {
if (schema == null) {
return getJdbcSchemaName(getDefaultSchema());
} else {
return getJdbcSchemaName(new CatalogAndSchema(schema.getCatalogName(), schema.getName()));
}
}
protected void init2ndPassNamesWithDefaults() {
if (tableName == null) {
tableName = DaoUtil.dbName(className);
}
if (classNameDao == null) {
classNameDao = className + "Dao";
}
if (classNameTest == null) {
classNameTest = className + "Test";
}
if (javaPackage == null) {
javaPackage = schema.getDefaultJavaPackage();
}
if (javaPackageDao == null) {
javaPackageDao = schema.getDefaultJavaPackageDao();
if (javaPackageDao == null) {
javaPackageDao = javaPackage;
}
}
if (javaPackageTest == null) {
javaPackageTest = schema.getDefaultJavaPackageTest();
if (javaPackageTest == null) {
javaPackageTest = javaPackage;
}
}
}
/**
* Initialization to be done after constructor assignments, such as setting of the all-important
* DDFManager.
*/
protected void initialize(
DDFManager manager,
Object data,
Class<?>[] typeSpecs,
String namespace,
String name,
Schema schema)
throws DDFException {
this.validateSchema(schema);
this.setManager(manager); // this must be done first in case later stuff needs a manager
if (typeSpecs != null) {
this.getRepresentationHandler().set(data, typeSpecs);
}
this.getSchemaHandler().setSchema(schema);
if (schema != null && schema.getTableName() == null) {
String tableName = this.getSchemaHandler().newTableName();
schema.setTableName(tableName);
}
if (Strings.isNullOrEmpty(namespace)) namespace = this.getManager().getNamespace();
this.setNamespace(namespace);
manager.setDDFUUID(this, UUID.randomUUID());
if (!Strings.isNullOrEmpty(name)) manager.setDDFName(this, name);
// Facades
this.ML = new MLFacade(this, this.getMLSupporter());
this.VIEWS = new ViewsFacade(this, this.getViewHandler());
this.Transform = new TransformFacade(this, this.getTransformationHandler());
this.R = new RFacade(this, this.getAggregationHandler());
this.mCreatedTime = new Date();
}
/** Returns the root schema. */
public static Schema root(Schema schema) {
for (Schema s = schema; ; ) {
Schema previous = s;
s = s.getParentSchema();
if (s == null) {
return previous;
}
}
}
void init2ndPass() {
init2ndPassNamesWithDefaults();
for (int i = 0; i < properties.size(); i++) {
Property property = properties.get(i);
property.setOrdinal(i);
property.init2ndPass();
if (property.isPrimaryKey()) {
propertiesPk.add(property);
} else {
propertiesNonPk.add(property);
}
}
if (propertiesPk.size() == 1) {
pkProperty = propertiesPk.get(0);
pkType = schema.mapToJavaTypeNullable(pkProperty.getPropertyType());
} else {
pkType = "Void";
}
propertiesColumns = new ArrayList<Property>(properties);
for (ToOne toOne : toOneRelations) {
toOne.init2ndPass();
Property[] fkProperties = toOne.getFkProperties();
for (Property fkProperty : fkProperties) {
if (!propertiesColumns.contains(fkProperty)) {
propertiesColumns.add(fkProperty);
}
}
}
for (ToMany toMany : toManyRelations) {
toMany.init2ndPass();
// Source Properties may not be virtual, so we do not need the following code:
// for (Property sourceProperty : toMany.getSourceProperties()) {
// if (!propertiesColumns.contains(sourceProperty)) {
// propertiesColumns.add(sourceProperty);
// }
// }
}
if (active == null) {
active = schema.isUseActiveEntitiesByDefault();
}
active |= !toOneRelations.isEmpty() || !toManyRelations.isEmpty();
if (hasKeepSections == null) {
hasKeepSections = schema.isHasKeepSectionsByDefault();
}
init2ndPassIndexNamesWithDefaults();
for (ContentProvider contentProvider : contentProviders) {
contentProvider.init2ndPass();
}
}
// ////// MetaData that deserves to be right here at the top level ////////
private void validateSchema(Schema schema) throws DDFException {
Set<String> columnSet = new HashSet<String>();
if (schema != null && schema.getColumns() != null) {
for (Column column : schema.getColumns()) {
if (columnSet.contains(column.getName())) {
throw new DDFException(String.format("Duplicated column name %s", column.getName()));
} else {
columnSet.add(column.getName());
}
}
}
}
/** Returns the path of a schema, appending the name of a table if not null. */
public static List<String> path(Schema schema, String name) {
final List<String> list = new ArrayList<String>();
if (name != null) {
list.add(name);
}
for (Schema s = schema; s != null; s = s.getParentSchema()) {
if (s.getParentSchema() != null || !s.getName().equals("")) {
// Omit the root schema's name from the path if it's the empty string,
// which it usually is.
list.add(s.getName());
}
}
return ImmutableList.copyOf(Lists.reverse(list));
}
/** It asserts the equality between an original table desc and a restored table desc. */
private static void assertSchemaEquality(String tableName, Schema schema)
throws IOException, TajoException {
Path path = new Path(CommonTestingUtil.getTestDir(), tableName);
TableDesc tableDesc =
new TableDesc(
IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
schema,
"TEXT",
new KeyValueSet(),
path.toUri());
// schema creation
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
catalog.createTable(tableDesc);
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
// change it for the equals test.
schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
assertEquals(schema, restored.getSchema());
// drop test
catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Schema schema = (Schema) o;
if (getCatalog() != null
? !getCatalog().equals(schema.getCatalog())
: schema.getCatalog() != null) return false;
if (getName() != null
? !getName().equalsIgnoreCase(schema.getName())
: schema.getName() != null) return false;
return true;
}
public final Table getTable(String tableName, boolean caseSensitive) {
if (caseSensitive) {
return compositeTableMap.get(tableName);
} else {
final TableEntry tableEntry = tableMapInsensitive.get(tableName);
if (tableEntry != null) {
return tableEntry.getTable();
}
final FunctionEntry entry = nullaryFunctionMapInsensitive.get(tableName);
if (entry != null) {
return ((TableMacro) entry.getFunction()).apply(ImmutableList.of());
}
for (String name : schema.getTableNames()) {
if (name.equalsIgnoreCase(tableName)) {
return schema.getTable(name);
}
}
return null;
}
}
/** Creates a ListTable. */
public ListTable(
Schema schema,
Type elementType,
RelDataType relDataType,
Expression expression,
List<T> list) {
super(schema.getQueryProvider(), elementType, expression);
this.schema = schema;
this.relDataType = relDataType;
this.list = list;
}
/** Returns the expression to access a table within a schema. */
public static Expression tableExpression(
Schema schema, Type elementType, String tableName, Class clazz) {
final MethodCallExpression expression;
if (Table.class.isAssignableFrom(clazz)) {
expression =
Expressions.call(
schema.getExpression(),
BuiltinMethod.SCHEMA_GET_TABLE.method,
Expressions.constant(tableName));
} else {
expression =
Expressions.call(
BuiltinMethod.SCHEMAS_QUERYABLE.method,
DataContext.ROOT,
schema.getExpression(),
Expressions.constant(elementType),
Expressions.constant(tableName));
}
return Types.castIfNecessary(clazz, expression);
}
/**
* Returns the unique metric values for each column.
*
* <p>The original unique values cannot be used because after aggregation, we almost certainly
* have new values to encode that were not present in the original data set.
*/
private Map<String, Set<Object>> computeUniqueMetricValues() {
Map<String, Set<Object>> uniqueMetricValues = new HashMap<String, Set<Object>>();
Iterator<StarTreeTableRow> tableIterator = starTreeBuilder.getTable().getAllCombinations();
while (tableIterator.hasNext()) {
StarTreeTableRow row = tableIterator.next();
for (int i = 0; i < schema.getMetricNames().size(); i++) {
String metricName = schema.getMetricNames().get(i);
Object metricValue = row.getMetrics().get(i);
Set<Object> uniqueValues = uniqueMetricValues.get(metricName);
if (uniqueValues == null) {
uniqueValues = new HashSet<Object>();
uniqueMetricValues.put(metricName, uniqueValues);
}
uniqueValues.add(metricValue);
}
}
return uniqueMetricValues;
}
public List<SqlOperator> lookupOperatorOverloads(
SqlIdentifier opName, SqlFunctionCategory category, SqlSyntax syntax) {
if (syntax != SqlSyntax.Function) {
return Collections.emptyList();
}
// FIXME: ignoring prefix of opName
String name = opName.names[opName.names.length - 1];
List<TableFunction> tableFunctions = rootSchema.getTableFunctions(name);
if (tableFunctions.isEmpty()) {
return Collections.emptyList();
}
return toOps(name, tableFunctions);
}
/** Converts a raw row into its (possibly partial) dimension and complete metric values */
private StarTreeTableRow extractValues(GenericRow row) {
List<Integer> dimensions = new ArrayList<Integer>();
for (String dimensionName : schema.getDimensionNames()) {
Integer valueId;
if (schema.getFieldSpecFor(dimensionName).isSingleValueField()
&& !starTreeIndexSpec.getExcludedDimensions().contains(dimensionName)) {
Object value = row.getValue(dimensionName);
valueId = dictionaryCreatorMap.get(dimensionName).indexOfSV(value);
} else {
// Multi-value fields are not supported - always ALL
valueId = V1Constants.STARTREE_ALL_NUMBER.intValue();
}
dimensions.add(valueId);
}
List<Number> metrics = new ArrayList<Number>(schema.getMetricNames().size());
for (MetricFieldSpec metricFieldSpec : schema.getMetricFieldSpecs()) {
Object value = row.getValue(metricFieldSpec.getName());
switch (metricFieldSpec.getDataType()) {
case INT:
metrics.add((Integer) value);
break;
case LONG:
metrics.add((Long) value);
break;
case DOUBLE:
metrics.add((Double) value);
break;
case FLOAT:
metrics.add((Float) value);
break;
default:
throw new IllegalStateException("Unsupported data type " + metricFieldSpec.getDataType());
}
}
return new StarTreeTableRow(dimensions, metrics);
}
private Set<HookConfiguration> assembleConfig(
final Map<String, String> hookConfig, final HookTemplate template) {
final Set<HookConfiguration> configuration = new HashSet<>();
final Set<Schema> fields = template.getSchema();
for (final Entry<String, String> configEntry : hookConfig.entrySet()) {
for (final Schema field : fields) {
final String fieldName = field.getFieldName();
if (fieldName.equalsIgnoreCase(configEntry.getKey())) {
final HookConfiguration config =
HookConfiguration.createNewWithoutHook(
field.getFieldType(), configEntry.getKey(), configEntry.getValue());
configuration.add(config);
break;
}
}
}
return configuration;
}
/** Returns the expression for a sub-schema. */
public static Expression subSchemaExpression(Schema schema, String name, Class type) {
// (Type) schemaExpression.getSubSchema("name")
Expression call =
Expressions.call(
schema.getExpression(),
BuiltinMethod.SCHEMA_GET_SUB_SCHEMA.method,
Expressions.constant(name));
// CHECKSTYLE: IGNORE 2
//noinspection unchecked
if (false && type != null && !type.isAssignableFrom(Schema.class)) {
return unwrap(call, type);
}
return call;
}
/** Creates an ArrayTable. */
public ArrayTable(
Schema schema,
Type elementType,
RelDataType relDataType,
Expression expression,
List<Pair<Representation, Object>> pairs,
int size) {
super(schema.getQueryProvider(), elementType, expression);
this.schema = schema;
this.relDataType = relDataType;
this.pairs = pairs;
this.size = size;
assert relDataType.getFieldCount() == pairs.size();
}
/**
* @param manager
* @param data
* @param typeSpecs
* @param namespace
* @param name
* @param schema
* @param tableName: name of the underlying table that representing the DDF
* @throws DDFException
*/
protected void initialize(
DDFManager manager,
Object data,
Class<?>[] typeSpecs,
String namespace,
String name,
Schema schema,
String tableName)
throws DDFException {
initialize(manager, data, typeSpecs, namespace, name, schema);
if (schema != null && tableName != null) {
schema.setTableName(tableName);
}
}
@Override
protected PreparedExecution implement(
RelDataType rowType,
RelNode rootRel,
SqlKind sqlKind,
ClassDeclaration decl,
Argument[] args) {
RelDataType resultType = rootRel.getRowType();
boolean isDml = sqlKind.belongsTo(SqlKind.DML);
javaCompiler = createCompiler();
EnumerableRelImplementor relImplementor =
getRelImplementor(rootRel.getCluster().getRexBuilder());
BlockExpression expr = relImplementor.implementRoot((EnumerableRel) rootRel);
ParameterExpression root0 = Expressions.parameter(DataContext.class, "root0");
String s =
Expressions.toString(
Blocks.create(
Expressions.declare(
Modifier.FINAL, (ParameterExpression) schema.getExpression(), root0),
expr),
false);
final Executable executable;
try {
executable =
(Executable)
ExpressionEvaluator.createFastScriptEvaluator(
s, Executable.class, new String[] {root0.name});
} catch (Exception e) {
throw Helper.INSTANCE.wrap("Error while compiling generated Java code:\n" + s, e);
}
if (timingTracer != null) {
timingTracer.traceTime("end codegen");
}
if (timingTracer != null) {
timingTracer.traceTime("end compilation");
}
return new PreparedExecution(
null, rootRel, resultType, isDml, mapTableModOp(isDml, sqlKind), null) {
public Object execute() {
return executable.execute(schema);
}
};
}
@Override
public void indexRow(GenericRow row) {
// Find matching leaves in StarTree for row
currentMatchingNodes.clear();
StarTreeTableRow tableRow = extractValues(row);
findMatchingLeaves(starTreeBuilder.getTree(), tableRow.getDimensions(), currentMatchingNodes);
// Only write the raw value, maintaining sort order (we will write aggregates when sealing)
for (StarTreeIndexNode node : currentMatchingNodes) {
Map<Integer, Integer> pathValues = node.getPathValues();
if (!pathValues.containsValue(StarTreeIndexNode.all())) {
StarTreeTableRange range = starTreeBuilder.getDocumentIdRange(node.getNodeId());
StarTreeTable subTable =
starTreeBuilder.getTable().view(range.getStartDocumentId(), range.getDocumentCount());
Integer nextMatchingDocumentId =
starTreeBuilder.getNextDocumentId(tableRow.getDimensions());
if (nextMatchingDocumentId == null) {
throw new IllegalStateException("Could not assign document ID for row " + tableRow);
}
// Write using that document ID to all columns
for (final String column : dictionaryCreatorMap.keySet()) {
Object columnValueToIndex = row.getValue(column);
if (schema.getFieldSpecFor(column).isSingleValueField()) {
System.out.println(column + ": " + columnValueToIndex);
int dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
((SingleValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap
.get(column)
.add(nextMatchingDocumentId, (Object) dictionaryIndex);
}
} else {
int[] dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
((MultiValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap.get(column).add(nextMatchingDocumentId, dictionaryIndex);
}
}
}
}
}
}
/**
* Re-initializes only the metric dictionaries using the unique metric values (computed after
* aggregation).
*/
private void resetMetricDictionaries(Map<String, Set<Object>> uniqueMetricValues)
throws Exception {
for (MetricFieldSpec spec : schema.getMetricFieldSpecs()) {
String column = spec.getName();
ColumnIndexCreationInfo info = columnInfo.get(column);
// The new unique values
Object[] valuesWithAggregates = uniqueMetricValues.get(column).toArray();
Arrays.sort(valuesWithAggregates);
// Reset dictionaries
dictionaryCreatorMap.put(
column,
new SegmentDictionaryCreator(info.hasNulls(), valuesWithAggregates, spec, outDir));
dictionaryCreatorMap.get(column).build();
}
}
/**
* Returns the user-defined split order, or dimensions in order of descending cardinality (removes
* excludes too)
*/
private List<String> computeSplitOrder(final Map<String, ColumnIndexCreationInfo> columnInfo) {
List<String> splitOrder;
if (starTreeIndexSpec.getSplitOrder() == null) {
splitOrder = new ArrayList<String>(schema.getDimensionNames());
Collections.sort(
splitOrder,
new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
Object[] o1UniqueElements = columnInfo.get(o1).getSortedUniqueElementsArray();
Object[] o2UniqueElements = columnInfo.get(o2).getSortedUniqueElementsArray();
return o2UniqueElements.length - o1UniqueElements.length; // descending
}
});
} else {
splitOrder = new ArrayList<String>(starTreeIndexSpec.getSplitOrder());
}
if (starTreeIndexSpec.getSplitExcludes() != null) {
splitOrder.removeAll(starTreeIndexSpec.getSplitExcludes());
}
return splitOrder;
}
@SuppressWarnings("all")
public class TestSerializerSample extends SpecificRecordBase implements SpecificRecord {
private static final long serialVersionUID = 1L;
public static final Schema SCHEMA =
Schema.parse(
"{\"type\":\"record\",\"name\":\"TestSerializerSample\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"int1\",\"type\":[\"int\",\"null\"]},{\"name\":\"tinyint1\",\"type\":[\"int\",\"null\"]},{\"name\":\"smallint1\",\"type\":[\"int\",\"null\"]},{\"name\":\"bigint1\",\"type\":[\"long\",\"null\"]},{\"name\":\"boolean1\",\"type\":[\"boolean\",\"null\"]},{\"name\":\"double1\",\"type\":[\"double\",\"null\"]},{\"name\":\"string1\",\"type\":[\"string\",\"null\"]},{\"name\":\"record\",\"type\":[{\"type\":\"record\",\"name\":\"Record\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"sInt\",\"type\":[\"int\",\"null\"]},{\"name\":\"sBoolean\",\"type\":[\"boolean\",\"null\"]},{\"name\":\"sString\",\"type\":[\"string\",\"null\"]}]},\"null\"]},{\"name\":\"list1\",\"type\":[{\"type\":\"array\",\"items\":\"string\"},\"null\"]},{\"name\":\"map1\",\"type\":[{\"type\":\"map\",\"values\":\"int\"},\"null\"]},{\"name\":\"enum1\",\"type\":[{\"type\":\"enum\",\"name\":\"Enum1Values\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"symbols\":[\"BLUE\",\"RED\",\"GREEN\"]},\"null\"]},{\"name\":\"nullableint\",\"type\":[\"int\",\"null\"]},{\"name\":\"bytes1\",\"type\":[\"bytes\",\"null\"]},{\"name\":\"container1\",\"type\":[{\"type\":\"record\",\"name\":\"Record2Container\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"record2list\",\"type\":[{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"Record2\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"enum2\",\"type\":[{\"type\":\"enum\",\"name\":\"Enum2Values\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"symbols\":[\"CAR\",\"BIKE\",\"PLANE\"]},\"null\"]},{\"name\":\"bigint2\",\"type\":[\"long\",\"null\"]},{\"name\":\"nullablebigint\",\"type\":[\"long\",\"null\"]},{\"name\":\"list2\",\"type\":[{\"type\":\"array\",\"items\":\"int\"},\"null\"]},{\"name\":\"map2\",\"type\":[{\"type\":\"map\",\"values\":\"Record\"},\"null\"]},{\"name\":\"byteslist\",\"type\":[{\"type\":\"array\",\"items\":\"bytes\"},\"null\"]},{\"name\":\"filling\",\"type\":[{\"type\":\"record\",\"name\":\"ModelFilling\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"stringfilling1\",\"type\":[\"string\",\"null\"]},{\"name\":\"stringfilling2\",\"type\":[\"string\",\"null\"]},{\"name\":\"stringfilling3\",\"type\":[\"string\",\"null\"]},{\"name\":\"stringfilling4\",\"type\":[\"string\",\"null\"]},{\"name\":\"intfilling\",\"type\":[\"int\",\"null\"]},{\"name\":\"boolfilling\",\"type\":[\"boolean\",\"null\"]},{\"name\":\"modelfilling\",\"type\":[{\"type\":\"record\",\"name\":\"ModelFilling2\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"longfilling\",\"type\":[\"long\",\"null\"]},{\"name\":\"stringfilling\",\"type\":[\"string\",\"null\"]},{\"name\":\"listfilling\",\"type\":[{\"type\":\"array\",\"items\":\"string\"},\"null\"]},{\"name\":\"enumfilling\",\"type\":[\"Enum2Values\",\"null\"]}]},\"null\"]},{\"name\":\"modelfilling3\",\"type\":[{\"type\":\"record\",\"name\":\"ModelFilling3\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.crosstest\",\"doc\":null,\"fields\":[{\"name\":\"intfilling\",\"type\":[\"int\",\"null\"]},{\"name\":\"doublefilling\",\"type\":[\"double\",\"null\"]}]},\"null\"]},{\"name\":\"enumfilling\",\"type\":[\"Enum1Values\",\"null\"]}]},\"null\"]}]}},\"null\"]}]},\"null\"]},{\"name\":\"innerSample\",\"type\":[\"TestSerializerSample\",\"null\"]}]}");
@Override
public Schema getSchema() {
return SCHEMA;
}
public TestSerializerSample(
Integer int1,
Integer tinyint1,
Integer smallint1,
Long bigint1,
Boolean boolean1,
Double double1,
String string1,
Record record,
List<String> list1,
Map<String, Integer> map1,
Enum1Values enum1,
Integer nullableint,
byte[] bytes1,
Record2Container container1,
TestSerializerSample innerSample) {
this.int1 = int1;
this.tinyint1 = tinyint1;
this.smallint1 = smallint1;
this.bigint1 = bigint1;
this.boolean1 = boolean1;
this.double1 = double1;
this.string1 = string1;
this.record = record;
this.list1 = list1;
this.map1 = map1;
this.enum1 = enum1;
this.nullableint = nullableint;
this.bytes1 = bytes1;
this.container1 = container1;
this.innerSample = innerSample;
}
public TestSerializerSample() {}
public Integer int1;
public Integer tinyint1;
public Integer smallint1;
public Long bigint1;
public Boolean boolean1;
public Double double1;
public String string1;
public Record record;
public List<String> list1;
public Map<String, Integer> map1;
public Enum1Values enum1;
public Integer nullableint;
public byte[] bytes1;
public Record2Container container1;
public TestSerializerSample innerSample;
public Integer getInt1() {
return int1;
}
public void setInt1(final Integer int1) {
this.int1 = int1;
}
public Integer getTinyint1() {
return tinyint1;
}
public void setTinyint1(final Integer tinyint1) {
this.tinyint1 = tinyint1;
}
public Integer getSmallint1() {
return smallint1;
}
public void setSmallint1(final Integer smallint1) {
this.smallint1 = smallint1;
}
public Long getBigint1() {
return bigint1;
}
public void setBigint1(final Long bigint1) {
this.bigint1 = bigint1;
}
public Boolean isBoolean1() {
return boolean1;
}
public void setBoolean1(final Boolean boolean1) {
this.boolean1 = boolean1;
}
public Double getDouble1() {
return double1;
}
public void setDouble1(final Double double1) {
this.double1 = double1;
}
public String getString1() {
return string1;
}
public void setString1(final String string1) {
this.string1 = string1;
}
public Record getRecord() {
return record;
}
public void setRecord(final Record record) {
this.record = record;
}
public List<String> getList1() {
return list1;
}
public void setList1(final List<String> list1) {
this.list1 = list1;
}
public Map<String, Integer> getMap1() {
return map1;
}
public void setMap1(final Map<String, Integer> map1) {
this.map1 = map1;
}
public Enum1Values getEnum1() {
return enum1;
}
public void setEnum1(final Enum1Values enum1) {
this.enum1 = enum1;
}
public Integer getNullableint() {
return nullableint;
}
public void setNullableint(final Integer nullableint) {
this.nullableint = nullableint;
}
public byte[] getBytes1() {
return bytes1;
}
public void setBytes1(final byte[] bytes1) {
this.bytes1 = bytes1;
}
public Record2Container getContainer1() {
return container1;
}
public void setContainer1(final Record2Container container1) {
this.container1 = container1;
}
public TestSerializerSample getInnerSample() {
return innerSample;
}
public void setInnerSample(final TestSerializerSample innerSample) {
this.innerSample = innerSample;
}
// Used by DatumWriter. Applications should not call.
public java.lang.Object get(int fieldPos) {
switch (fieldPos) {
case 0:
return this.int1;
case 1:
return this.tinyint1;
case 2:
return this.smallint1;
case 3:
return this.bigint1;
case 4:
return this.boolean1;
case 5:
return this.double1;
case 6:
return this.string1;
case 7:
return this.record;
case 8:
return this.list1;
case 9:
return this.map1;
case 10:
return this.enum1;
case 11:
return this.nullableint;
case 12:
return this.bytes1;
case 13:
return this.container1;
case 14:
return this.innerSample;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in get()");
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value = "unchecked")
public void put(int fieldPos, java.lang.Object fieldValue) {
switch (fieldPos) {
case 0:
this.int1 = (Integer) fieldValue;
break;
case 1:
this.tinyint1 = (Integer) fieldValue;
break;
case 2:
this.smallint1 = (Integer) fieldValue;
break;
case 3:
this.bigint1 = (Long) fieldValue;
break;
case 4:
this.boolean1 = (Boolean) fieldValue;
break;
case 5:
this.double1 = (Double) fieldValue;
break;
case 6:
this.string1 = (String) fieldValue;
break;
case 7:
this.record = (Record) fieldValue;
break;
case 8:
this.list1 = (List<String>) fieldValue;
break;
case 9:
this.map1 = (Map<String, Integer>) fieldValue;
break;
case 10:
this.enum1 = (Enum1Values) fieldValue;
break;
case 11:
this.nullableint = (Integer) fieldValue;
break;
case 12:
this.bytes1 = (byte[]) fieldValue;
break;
case 13:
this.container1 = (Record2Container) fieldValue;
break;
case 14:
this.innerSample = (TestSerializerSample) fieldValue;
break;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in put()");
}
}
@Override
public boolean equals(Object obj) {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final TestSerializerSample other = (TestSerializerSample) obj;
return Objects.equal(this.int1, other.int1)
&& Objects.equal(this.tinyint1, other.tinyint1)
&& Objects.equal(this.smallint1, other.smallint1)
&& Objects.equal(this.bigint1, other.bigint1)
&& Objects.equal(this.boolean1, other.boolean1)
&& Objects.equal(this.double1, other.double1)
&& Objects.equal(this.string1, other.string1)
&& Objects.equal(this.record, other.record)
&& Objects.equal(this.list1, other.list1)
&& Objects.equal(this.map1, other.map1)
&& Objects.equal(this.enum1, other.enum1)
&& Objects.equal(this.nullableint, other.nullableint)
&& Arrays.equals(this.bytes1, other.bytes1)
&& Objects.equal(this.container1, other.container1)
&& Objects.equal(this.innerSample, other.innerSample);
}
@Override
public int hashCode() {
int result = 1;
result = 31 * result + (this.int1 == null ? 0 : this.int1.hashCode());
result = 31 * result + (this.tinyint1 == null ? 0 : this.tinyint1.hashCode());
result = 31 * result + (this.smallint1 == null ? 0 : this.smallint1.hashCode());
result = 31 * result + (this.bigint1 == null ? 0 : this.bigint1.hashCode());
result = 31 * result + (this.boolean1 == null ? 0 : this.boolean1.hashCode());
result = 31 * result + (this.double1 == null ? 0 : this.double1.hashCode());
result = 31 * result + (this.string1 == null ? 0 : this.string1.hashCode());
result = 31 * result + (this.record == null ? 0 : this.record.hashCode());
result = 31 * result + (this.list1 == null ? 0 : this.list1.hashCode());
result = 31 * result + (this.map1 == null ? 0 : this.map1.hashCode());
result = 31 * result + (this.enum1 == null ? 0 : this.enum1.hashCode());
result = 31 * result + (this.nullableint == null ? 0 : this.nullableint.hashCode());
result = 31 * result + (this.bytes1 == null ? 0 : Arrays.hashCode(this.bytes1));
result = 31 * result + (this.container1 == null ? 0 : this.container1.hashCode());
result = 31 * result + (this.innerSample == null ? 0 : this.innerSample.hashCode());
return result;
}
@Override
public String toString() {
return Objects.toStringHelper(this)
.add("int1", int1)
.add("tinyint1", tinyint1)
.add("smallint1", smallint1)
.add("bigint1", bigint1)
.add("boolean1", boolean1)
.add("double1", double1)
.add("string1", string1)
.add("record", record)
.add("list1", list1)
.add("map1", map1)
.add("enum1", enum1)
.add("nullableint", nullableint)
.add("bytes1", bytes1)
.add("container1", container1)
.add("innerSample", innerSample)
.toString();
}
}
@SuppressWarnings("all")
public class AddMessageRequestType extends SpecificRecordBase implements SpecificRecord {
private static final long serialVersionUID = 1L;
public static final Schema SCHEMA =
Schema.parse(
"{\"type\":\"record\",\"name\":\"AddMessageRequestType\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.message\",\"doc\":null,\"fields\":[{\"name\":\"message\",\"type\":[{\"type\":\"record\",\"name\":\"Message\",\"namespace\":\"com.ctriposs.baiji.rpc.samples.message\",\"doc\":null,\"fields\":[{\"name\":\"content\",\"type\":[\"string\",\"null\"]},{\"name\":\"author\",\"type\":[\"string\",\"null\"]},{\"name\":\"createdTime\",\"type\":[\"datetime\",\"null\"]}]},\"null\"]}]}");
@Override
public Schema getSchema() {
return SCHEMA;
}
public AddMessageRequestType(Message message) {
this.message = message;
}
public AddMessageRequestType() {}
public Message message;
public Message getMessage() {
return message;
}
public void setMessage(final Message message) {
this.message = message;
}
// Used by DatumWriter. Applications should not call.
public Object get(int fieldPos) {
switch (fieldPos) {
case 0:
return this.message;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in get()");
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value = "unchecked")
public void put(int fieldPos, Object fieldValue) {
switch (fieldPos) {
case 0:
this.message = (Message) fieldValue;
break;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in put()");
}
}
@Override
public boolean equals(Object obj) {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final AddMessageRequestType other = (AddMessageRequestType) obj;
return Objects.equal(this.message, other.message);
}
@Override
public int hashCode() {
int result = 1;
result = 31 * result + (this.message == null ? 0 : this.message.hashCode());
return result;
}
@Override
public String toString() {
return Objects.toStringHelper(this).add("message", message).toString();
}
}
public void driver() {
// get instance of MilliTimer() for benchmarking
MilliTimer timer = new MilliTimer();
// start timer
timer.startTimer();
// get instance of other helper objects
Config config = new Config("props/superD.properties");
config.loadConfig("props/log4j.properties");
log.info("\n\n");
log.info("Starting program!");
// CREATE DATABASE
H2 db = null;
try { // TODO get rid of this try/catch -- it covers the entire method
// and swallows everything that does not catch. not good...
db = Database.getInstance();
// Create CHECKDUPES OBJ
CheckDupes check = new CheckDupes();
// CONNECT TO DATABASE
try {
db.openConnection();
} catch (ClassNotFoundException | SQLException e) {
log.fatal("Failed to open database connection! Check config file!", e);
}
// LOAD DATABASE TABLES
Schema s = new Schema();
String sqlSchema = s.getSchema();
PreparedStatement psSchema = db.getConnection().prepareStatement(sqlSchema);
try {
log.info("Running schema update on db: " + db.getDbPath());
psSchema.execute();
log.info("Schema update complete!");
} catch (Exception e) {
e.printStackTrace();
}
try {
db.closeConnection();
} catch (SQLException e) {
log.warn("Failed to close database connection!", e);
}
// INSTEAD OF COMMAND LINE ARGUMENTS,
// PROMPT FOR VARIOUS SETTINGS IF
// USER WANTS TO DO THIS INSTEAD OF
// PROP FILE
// TODO need at least 1 cmd line argument to signal if user pompt is wanted or not...
// TODO otherwise automated runs are not possible even with props file used.
Scanner in = new Scanner(System.in);
System.out.println("Would you like to read from prop file or enter options now?");
System.out.print("Enter 1 to enter configuration now, 2 to use existing prop file: ");
int choice = 2;
try {
choice = in.nextInt();
} catch (Exception e) { // TODO find specific exceptions thrown by Scanner(System.in)
System.out.println("Invalid input! Proceeding using property file"); // TODO log out
}
if (choice == 1) {
readSetup();
}
// END PROMPT FOR COMMAND LINE ARGUMENTS
// Run Setup() to do main logic, make calls to Walk()
setup();
// DATABASE now filled with all file hashes; time to look for duplicates
check.checkDupes();
// ALL DONE! stop timer
timer.stopTimer();
log.info("Total Runtime: " + timer.getTime());
} catch (
Exception
e) { // TODO get rid of this to narrow try/catch scope for improved exception
// handling/recovery. log out
e.printStackTrace();
}
}
/**
* Shows the pagination data for the item search. Child elements include the page number returned,
* the maximum entries returned per page, the total number of pages that can be returned, and the
* total number of items that match the search criteria.
*/
@SuppressWarnings("all")
public class PaginationOutputType extends SpecificRecordBase implements SpecificRecord {
private static final long serialVersionUID = 1L;
public static final Schema SCHEMA =
Schema.parse(
"{\"type\":\"record\",\"name\":\"PaginationOutputType\",\"namespace\":\"com.ctriposs.baiji.rpc.common.types\",\"doc\":null,\"fields\":[{\"name\":\"pageNumber\",\"type\":[\"int\",\"null\"]},{\"name\":\"entriesPerPage\",\"type\":[\"int\",\"null\"]},{\"name\":\"totalPages\",\"type\":[\"int\",\"null\"]},{\"name\":\"totalEntries\",\"type\":[\"int\",\"null\"]}]}");
@Override
public Schema getSchema() {
return SCHEMA;
}
public PaginationOutputType(
Integer pageNumber, Integer entriesPerPage, Integer totalPages, Integer totalEntries) {
this.pageNumber = pageNumber;
this.entriesPerPage = entriesPerPage;
this.totalPages = totalPages;
this.totalEntries = totalEntries;
}
public PaginationOutputType() {}
/**
* The subset of item data returned in the current response. Search results are divided into sets,
* or "pages," of item data. The number of pages is equal to the total number of items matching
* the search criteria divided by the value specified for entriesPerPage in the request. The
* response for a request contains one "page" of item data. This returned value indicates the page
* number of item data returned (a subset of the complete result set). If this field contains 1,
* the response contains the first page of item data (the default). If the value returned in
* totalEntries is less than the value for entriesPerPage, pageNumber returns 1 and the response
* contains the entire result set. The value of pageNumber is normally equal to the value input
* for paginationInput.pageNumber. However, if the number input for pageNumber is greater than the
* total possible pages of output, Baiji returns the last page of item data in the result set, and
* the value for pageNumber is set to the respective (last) page number.
*/
public Integer pageNumber;
/**
* The maximum number of items that can be returned in the response. This number is always equal
* to the value input for paginationInput.entriesPerPage. The end of the result set has been
* reached if the number specified for entriesPerPage is greater than the number of items found on
* the specified pageNumber. In this case, there will be fewer items returned than the number
* specified in entriesPerPage. This can be determined by comparing the entriesPerPage value with
* the value returned in the count attribute for the searchResult field.
*/
public Integer entriesPerPage;
/**
* The total number of pages of data that could be returned by repeated search requests. Note that
* if you modify the value of inputPagination.entriesPerPage in a request, the value output for
* totalPages will change. A value of "0" is returned if service does not find any items that
* match the search criteria.
*/
public Integer totalPages;
/**
* The total number of items found that match the search criteria in your request. Depending on
* the input value for entriesPerPage, the response might include only a portion (a page) of the
* entire result set. A value of "0" is returned if service does not find any items that match the
* search criteria.
*/
public Integer totalEntries;
/**
* The subset of item data returned in the current response. Search results are divided into sets,
* or "pages," of item data. The number of pages is equal to the total number of items matching
* the search criteria divided by the value specified for entriesPerPage in the request. The
* response for a request contains one "page" of item data. This returned value indicates the page
* number of item data returned (a subset of the complete result set). If this field contains 1,
* the response contains the first page of item data (the default). If the value returned in
* totalEntries is less than the value for entriesPerPage, pageNumber returns 1 and the response
* contains the entire result set. The value of pageNumber is normally equal to the value input
* for paginationInput.pageNumber. However, if the number input for pageNumber is greater than the
* total possible pages of output, Baiji returns the last page of item data in the result set, and
* the value for pageNumber is set to the respective (last) page number.
*/
public Integer getPageNumber() {
return pageNumber;
}
/**
* The subset of item data returned in the current response. Search results are divided into sets,
* or "pages," of item data. The number of pages is equal to the total number of items matching
* the search criteria divided by the value specified for entriesPerPage in the request. The
* response for a request contains one "page" of item data. This returned value indicates the page
* number of item data returned (a subset of the complete result set). If this field contains 1,
* the response contains the first page of item data (the default). If the value returned in
* totalEntries is less than the value for entriesPerPage, pageNumber returns 1 and the response
* contains the entire result set. The value of pageNumber is normally equal to the value input
* for paginationInput.pageNumber. However, if the number input for pageNumber is greater than the
* total possible pages of output, Baiji returns the last page of item data in the result set, and
* the value for pageNumber is set to the respective (last) page number.
*/
public void setPageNumber(final Integer pageNumber) {
this.pageNumber = pageNumber;
}
/**
* The maximum number of items that can be returned in the response. This number is always equal
* to the value input for paginationInput.entriesPerPage. The end of the result set has been
* reached if the number specified for entriesPerPage is greater than the number of items found on
* the specified pageNumber. In this case, there will be fewer items returned than the number
* specified in entriesPerPage. This can be determined by comparing the entriesPerPage value with
* the value returned in the count attribute for the searchResult field.
*/
public Integer getEntriesPerPage() {
return entriesPerPage;
}
/**
* The maximum number of items that can be returned in the response. This number is always equal
* to the value input for paginationInput.entriesPerPage. The end of the result set has been
* reached if the number specified for entriesPerPage is greater than the number of items found on
* the specified pageNumber. In this case, there will be fewer items returned than the number
* specified in entriesPerPage. This can be determined by comparing the entriesPerPage value with
* the value returned in the count attribute for the searchResult field.
*/
public void setEntriesPerPage(final Integer entriesPerPage) {
this.entriesPerPage = entriesPerPage;
}
/**
* The total number of pages of data that could be returned by repeated search requests. Note that
* if you modify the value of inputPagination.entriesPerPage in a request, the value output for
* totalPages will change. A value of "0" is returned if service does not find any items that
* match the search criteria.
*/
public Integer getTotalPages() {
return totalPages;
}
/**
* The total number of pages of data that could be returned by repeated search requests. Note that
* if you modify the value of inputPagination.entriesPerPage in a request, the value output for
* totalPages will change. A value of "0" is returned if service does not find any items that
* match the search criteria.
*/
public void setTotalPages(final Integer totalPages) {
this.totalPages = totalPages;
}
/**
* The total number of items found that match the search criteria in your request. Depending on
* the input value for entriesPerPage, the response might include only a portion (a page) of the
* entire result set. A value of "0" is returned if service does not find any items that match the
* search criteria.
*/
public Integer getTotalEntries() {
return totalEntries;
}
/**
* The total number of items found that match the search criteria in your request. Depending on
* the input value for entriesPerPage, the response might include only a portion (a page) of the
* entire result set. A value of "0" is returned if service does not find any items that match the
* search criteria.
*/
public void setTotalEntries(final Integer totalEntries) {
this.totalEntries = totalEntries;
}
// Used by DatumWriter. Applications should not call.
public java.lang.Object get(int fieldPos) {
switch (fieldPos) {
case 0:
return this.pageNumber;
case 1:
return this.entriesPerPage;
case 2:
return this.totalPages;
case 3:
return this.totalEntries;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in get()");
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value = "unchecked")
public void put(int fieldPos, java.lang.Object fieldValue) {
switch (fieldPos) {
case 0:
this.pageNumber = (Integer) fieldValue;
break;
case 1:
this.entriesPerPage = (Integer) fieldValue;
break;
case 2:
this.totalPages = (Integer) fieldValue;
break;
case 3:
this.totalEntries = (Integer) fieldValue;
break;
default:
throw new BaijiRuntimeException("Bad index " + fieldPos + " in put()");
}
}
@Override
public boolean equals(Object obj) {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final PaginationOutputType other = (PaginationOutputType) obj;
return Objects.equal(this.pageNumber, other.pageNumber)
&& Objects.equal(this.entriesPerPage, other.entriesPerPage)
&& Objects.equal(this.totalPages, other.totalPages)
&& Objects.equal(this.totalEntries, other.totalEntries);
}
@Override
public int hashCode() {
int result = 1;
result = 31 * result + (this.pageNumber == null ? 0 : this.pageNumber.hashCode());
result = 31 * result + (this.entriesPerPage == null ? 0 : this.entriesPerPage.hashCode());
result = 31 * result + (this.totalPages == null ? 0 : this.totalPages.hashCode());
result = 31 * result + (this.totalEntries == null ? 0 : this.totalEntries.hashCode());
return result;
}
@Override
public String toString() {
return Objects.toStringHelper(this)
.add("pageNumber", pageNumber)
.add("entriesPerPage", entriesPerPage)
.add("totalPages", totalPages)
.add("totalEntries", totalEntries)
.toString();
}
}
private void validateHookRules(
final HookTemplate template, final Set<HookConfiguration> config, Set<HookResource> events) {
final List<ApiParameterError> dataValidationErrors = new ArrayList<>();
final DataValidatorBuilder baseDataValidator =
new DataValidatorBuilder(dataValidationErrors).resource("hook");
if (!template.getName().equalsIgnoreCase(webTemplateName)
&& this.hookRepository.findOneByTemplateId(template.getId()) != null) {
final String errorMessage = "multiple.non.web.template.hooks.not.supported";
baseDataValidator.reset().failWithCodeNoParameterAddedToErrorCode(errorMessage);
}
for (final HookConfiguration conf : config) {
final String fieldValue = conf.getFieldValue();
if (conf.getFieldName().equals(contentTypeName)) {
if (!(fieldValue.equalsIgnoreCase("json") || fieldValue.equalsIgnoreCase("form"))) {
final String errorMessage = "content.type.must.be.json.or.form";
baseDataValidator.reset().failWithCodeNoParameterAddedToErrorCode(errorMessage);
}
}
if (conf.getFieldName().equals(payloadURLName)) {
try {
final WebHookService service = ProcessorHelper.createWebHookService(fieldValue);
service.sendEmptyRequest();
} catch (RetrofitError re) {
// Swallow error if it's because of method not supported or
// if url throws 404 - required for integration test,
// url generated on 1st POST request
if (re.getResponse() == null) {
String errorMessage = "url.invalid";
baseDataValidator.reset().failWithCodeNoParameterAddedToErrorCode(errorMessage);
}
}
}
}
if (events == null || events.isEmpty()) {
final String errorMessage = "registered.events.cannot.be.empty";
baseDataValidator.reset().failWithCodeNoParameterAddedToErrorCode(errorMessage);
}
final Set<Schema> fields = template.getSchema();
for (final Schema field : fields) {
if (!field.isOptional()) {
boolean found = false;
for (final HookConfiguration conf : config) {
if (field.getFieldName().equals(conf.getFieldName())) {
found = true;
}
}
if (!found) {
final String errorMessage = "required.config.field." + "not.provided";
baseDataValidator
.reset()
.value(field.getFieldName())
.failWithCodeNoParameterAddedToErrorCode(errorMessage);
}
}
}
if (!dataValidationErrors.isEmpty()) {
throw new PlatformApiDataValidationException(dataValidationErrors);
}
}
