/**
* get the real column properties from a possible array mapping, keeping most of this stuff inside
* "inner"
*/
private Map<String, Object> furtherColumnProperties(Map<String, Object> columnProperties) {
if (columnProperties.get("inner") != null) {
return (Map<String, Object>) columnProperties.get("inner");
} else {
return columnProperties;
}
}
public SmartNameFieldMappers smartName(String smartName, @Nullable String[] types) {
if (types == null || types.length == 0) {
return smartName(smartName);
}
if (types.length == 1 && types[0].equals("_all")) {
return smartName(smartName);
}
for (String type : types) {
DocumentMapper documentMapper = mappers.get(type);
// we found a mapper
if (documentMapper != null) {
// see if we find a field for it
FieldMappers mappers = documentMapper.mappers().smartName(smartName);
if (mappers != null) {
return new SmartNameFieldMappers(this, mappers, documentMapper, false);
}
}
}
// did not find explicit field in the type provided, see if its prefixed with type
int dotIndex = smartName.indexOf('.');
if (dotIndex != -1) {
String possibleType = smartName.substring(0, dotIndex);
DocumentMapper possibleDocMapper = mappers.get(possibleType);
if (possibleDocMapper != null) {
String possibleName = smartName.substring(dotIndex + 1);
FieldMappers mappers = possibleDocMapper.mappers().smartName(possibleName);
if (mappers != null) {
return new SmartNameFieldMappers(this, mappers, possibleDocMapper, true);
}
}
}
// we did not find the field mapping in any of the types, so don't go and try to find
// it in other types...
return null;
}
public SmartNameObjectMapper smartNameObjectMapper(String smartName, @Nullable String[] types) {
if (types == null || types.length == 0) {
return smartNameObjectMapper(smartName);
}
if (types.length == 1 && types[0].equals("_all")) {
return smartNameObjectMapper(smartName);
}
for (String type : types) {
DocumentMapper possibleDocMapper = mappers.get(type);
if (possibleDocMapper != null) {
ObjectMapper mapper = possibleDocMapper.objectMappers().get(smartName);
if (mapper != null) {
return new SmartNameObjectMapper(mapper, possibleDocMapper);
}
}
}
// did not find one, see if its prefixed by type
int dotIndex = smartName.indexOf('.');
if (dotIndex != -1) {
String possibleType = smartName.substring(0, dotIndex);
DocumentMapper possibleDocMapper = mappers.get(possibleType);
if (possibleDocMapper != null) {
String possiblePath = smartName.substring(dotIndex + 1);
ObjectMapper mapper = possibleDocMapper.objectMappers().get(possiblePath);
if (mapper != null) {
return new SmartNameObjectMapper(mapper, possibleDocMapper);
}
}
}
// did not explicitly find one under the types provided, or prefixed by type...
return null;
}
private Map<String, Tag> findLastParent(final String[] parts) {
Map<String, Tag> map = NBTStorage.this.root;
for (int i = 0; i < parts.length - 1; ++i) {
if (!map.containsKey(parts[i]) || !(map.get(parts[i]) instanceof CompoundTag)) {
return null;
}
map = map.get(parts[i]).getValue();
}
return map;
}
// purchase product and return relevant product recommendations
public List<Product> purchase(Customer customer, Product product) {
purchasesCache.put(customer.id, product.id);
ProductAssociativityGraph graph = productAssociativityGraphMap.get(product.category);
Vertex v = Vertex.create(product.id);
List<Vertex> associations = graph.getProductAssociations(v);
int recommendSize = Math.min(associations.size(), maxNumRecommendations);
return associations
.stream()
.map(vertex -> productCache.get(vertex.productId))
.limit(recommendSize)
.collect(Collectors.toList());
}
/** extracts index definitions as well */
@SuppressWarnings("unchecked")
private void internalExtractColumnDefinitions(
ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
if (propertiesMap == null) {
return;
}
for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
Map<String, Object> columnProperties = (Map) columnEntry.getValue();
DataType columnDataType = getColumnDataType(columnProperties);
ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
columnProperties = furtherColumnProperties(columnProperties);
ReferenceInfo.IndexType columnIndexType = getColumnIndexType(columnProperties);
if (columnDataType == DataTypes.OBJECT
|| (columnDataType.id() == ArrayType.ID
&& ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
ColumnPolicy columnPolicy = ColumnPolicy.of(columnProperties.get("dynamic"));
add(newIdent, columnDataType, columnPolicy, ReferenceInfo.IndexType.NO, false);
if (columnProperties.get("properties") != null) {
// walk nested
internalExtractColumnDefinitions(
newIdent, (Map<String, Object>) columnProperties.get("properties"));
}
} else if (columnDataType != DataTypes.NOT_SUPPORTED) {
List<String> copyToColumns = getNested(columnProperties, "copy_to");
// extract columns this column is copied to, needed for indices
if (copyToColumns != null) {
for (String copyToColumn : copyToColumns) {
ColumnIdent targetIdent = ColumnIdent.fromPath(copyToColumn);
IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(targetIdent);
builder.addColumn(
newInfo(newIdent, columnDataType, ColumnPolicy.DYNAMIC, columnIndexType));
}
}
// is it an index?
if (indicesMap.containsKey(newIdent.fqn())) {
IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(newIdent);
builder
.indexType(columnIndexType)
.ident(new ReferenceIdent(ident, newIdent))
.analyzer((String) columnProperties.get("analyzer"));
} else {
add(newIdent, columnDataType, columnIndexType);
}
}
}
}
private void putTag(final String key, final Tag tag) {
final String[] parts =
(String[])
Iterables.toArray(
Splitter.on('.').split((CharSequence) this.createRelativeKey(key)),
(Class) String.class);
Map<String, Tag> parent = NBTStorage.this.root;
for (int i = 0; i < parts.length - 1; ++i) {
if (!parent.containsKey(parts[i]) || !(parent.get(parts[i]) instanceof CompoundTag)) {
parent.put(parts[i], new CompoundTag(parts[i]));
}
parent = parent.get(parts[i]).getValue();
}
parent.put(tag.getName(), tag);
}
/** Extracts document and cluster lists before serialization. */
@Persist
private void beforeSerialization() {
/*
* See http://issues.carrot2.org/browse/CARROT-693; this monitor does not save us
* in multi-threaded environment anyway. A better solution would be to prepare
* this eagerly in the constructor, but we try to balance overhead and full
* correctness here.
*/
synchronized (this) {
query = (String) attributes.get(AttributeNames.QUERY);
if (getDocuments() != null) {
documents = Lists.newArrayList(getDocuments());
} else {
documents = null;
}
if (getClusters() != null) {
clusters = Lists.newArrayList(getClusters());
} else {
clusters = null;
}
otherAttributesForSerialization = MapUtils.asHashMap(SimpleXmlWrappers.wrap(attributes));
otherAttributesForSerialization.remove(AttributeNames.QUERY);
otherAttributesForSerialization.remove(AttributeNames.CLUSTERS);
otherAttributesForSerialization.remove(AttributeNames.DOCUMENTS);
if (otherAttributesForSerialization.isEmpty()) {
otherAttributesForSerialization = null;
}
}
}
private ImmutableList<ColumnIdent> getPrimaryKey() {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
Object pKeys = metaMap.get("primary_keys");
if (pKeys != null) {
if (pKeys instanceof String) {
builder.add(ColumnIdent.fromPath((String) pKeys));
return builder.build();
} else if (pKeys instanceof Collection) {
Collection keys = (Collection) pKeys;
if (!keys.isEmpty()) {
for (Object pkey : keys) {
builder.add(ColumnIdent.fromPath(pkey.toString()));
}
return builder.build();
}
}
}
}
if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
hasAutoGeneratedPrimaryKey = true;
return ImmutableList.of(ID_IDENT);
}
return ImmutableList.of();
}
// never expose this to the outside world, we need to reparse the doc mapper so we get fresh
// instances of field mappers to properly remove existing doc mapper
private DocumentMapper merge(DocumentMapper mapper) {
synchronized (typeMutex) {
if (mapper.type().length() == 0) {
throw new InvalidTypeNameException("mapping type name is empty");
}
if (mapper.type().charAt(0) == '_' && !PercolatorService.TYPE_NAME.equals(mapper.type())) {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] can't start with '_'");
}
if (mapper.type().contains("#")) {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] should not include '#' in it");
}
if (mapper.type().contains(",")) {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] should not include ',' in it");
}
if (mapper.type().contains(".")) {
logger.warn(
"Type [{}] contains a '.', it is recommended not to include it within a type name",
mapper.type());
}
// we can add new field/object mappers while the old ones are there
// since we get new instances of those, and when we remove, we remove
// by instance equality
DocumentMapper oldMapper = mappers.get(mapper.type());
if (oldMapper != null) {
DocumentMapper.MergeResult result = oldMapper.merge(mapper, mergeFlags().simulate(false));
if (result.hasConflicts()) {
// TODO: What should we do???
if (logger.isDebugEnabled()) {
logger.debug(
"merging mapping for type [{}] resulted in conflicts: [{}]",
mapper.type(),
Arrays.toString(result.conflicts()));
}
}
return oldMapper;
} else {
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
mapper.traverse(fieldMappersAgg);
addFieldMappers(
fieldMappersAgg.mappers.toArray(new FieldMapper[fieldMappersAgg.mappers.size()]));
mapper.addFieldMapperListener(fieldMapperListener, false);
ObjectMapperListener.Aggregator objectMappersAgg = new ObjectMapperListener.Aggregator();
mapper.traverse(objectMappersAgg);
addObjectMappers(
objectMappersAgg.mappers.toArray(new ObjectMapper[objectMappersAgg.mappers.size()]));
mapper.addObjectMapperListener(objectMapperListener, false);
mappers = newMapBuilder(mappers).put(mapper.type(), mapper).map();
for (DocumentTypeListener typeListener : typeListeners) {
typeListener.created(mapper.type());
}
return mapper;
}
}
}
/**
* Returns smart field mappers based on a smart name. A smart name is one that can optioannly be
* prefixed with a type (and then a '.'). If it is, then the {@link
* MapperService.SmartNameFieldMappers} will have the doc mapper set.
*
* <p>
*
* <p>It also (without the optional type prefix) try and find the {@link FieldMappers} for the
* specific name. It will first try to find it based on the full name (with the dots if its a
* compound name). If it is not found, will try and find it based on the indexName (which can be
* controlled in the mapping), and last, will try it based no the name itself.
*
* <p>
*
* <p>If nothing is found, returns null.
*/
public SmartNameFieldMappers smartName(String smartName) {
int dotIndex = smartName.indexOf('.');
if (dotIndex != -1) {
String possibleType = smartName.substring(0, dotIndex);
DocumentMapper possibleDocMapper = mappers.get(possibleType);
if (possibleDocMapper != null) {
String possibleName = smartName.substring(dotIndex + 1);
FieldMappers mappers = possibleDocMapper.mappers().smartName(possibleName);
if (mappers != null) {
return new SmartNameFieldMappers(this, mappers, possibleDocMapper, true);
}
}
}
FieldMappers fieldMappers = fullName(smartName);
if (fieldMappers != null) {
return new SmartNameFieldMappers(this, fieldMappers, null, false);
}
fieldMappers = indexName(smartName);
if (fieldMappers != null) {
return new SmartNameFieldMappers(this, fieldMappers, null, false);
}
fieldMappers = name(smartName);
if (fieldMappers != null) {
return new SmartNameFieldMappers(this, fieldMappers, null, false);
}
return null;
}
public double getFlagDouble(char ch, double def) throws NumberFormatException {
final String value = valueFlags.get(ch);
if (value == null) {
return def;
}
return Double.parseDouble(value);
}
public int getFlagInteger(char ch, int def) throws NumberFormatException {
final String value = valueFlags.get(ch);
if (value == null) {
return def;
}
return Integer.parseInt(value);
}
public String getFlag(char ch, String def) {
final String value = valueFlags.get(ch);
if (value == null) {
return def;
}
return value;
}
private IndexReferenceInfo.Builder getOrCreateIndexBuilder(ColumnIdent ident) {
IndexReferenceInfo.Builder builder = indicesBuilder.get(ident);
if (builder == null) {
builder = new IndexReferenceInfo.Builder();
indicesBuilder.put(ident, builder);
}
return builder;
}
private void repeatLabelsBindingForInstruction(
@NotNull Instruction originalInstruction,
@NotNull Map<Label, Label> originalToCopy,
@NotNull Multimap<Instruction, Label> originalLabelsForInstruction) {
for (Label originalLabel : originalLabelsForInstruction.get(originalInstruction)) {
bindLabel(originalToCopy.get(originalLabel));
}
}
private MapMessage newMapMessage(Map<String, Object> body) throws JMSException {
MapMessage message = new MapMessageImpl();
for (String key : body.keySet()) {
Object value = body.get(key);
message.setObject(key, value);
}
return message;
}
@Override
public TitanType getType(String name) {
verifyOpen();
TitanType type = typeCache.get(name);
if (type == null)
type = (TitanType) Iterables.getOnlyElement(getVertices(SystemKey.TypeName, name), null);
return type;
}
@Override
public PColumnFamily getColumnFamily(String familyName) throws ColumnFamilyNotFoundException {
PColumnFamily family = familyByString.get(familyName);
if (family == null) {
throw new ColumnFamilyNotFoundException(familyName);
}
return family;
}
@Override
public PColumnFamily getColumnFamily(byte[] familyBytes) throws ColumnFamilyNotFoundException {
PColumnFamily family = familyByBytes.get(familyBytes);
if (family == null) {
String familyName = Bytes.toString(familyBytes);
throw new ColumnFamilyNotFoundException(familyName);
}
return family;
}
private static void test() {
String customerKey = "customer";
String productKey = "product";
IntStream.range(0, 5)
.forEach(
index ->
customerCache.put(
customerKey + index, new Customer(customerKey + index, "name" + index)));
IntStream.range(0, 100)
.forEach(
index ->
productCache.put(
productKey + index, new Product(productKey + index, "category" + (index % 5))));
Random random = new Random();
Set<String> productIds = new HashSet<>();
int activeProductSize = productCache.size() / 10;
for (int i = 0; i < 1000; i++) {
Customer customer = customerCache.get(customerKey + random.nextInt(customerCache.size()));
int randomSuffix = random.nextInt(activeProductSize);
if (randomSuffix < 3) {
randomSuffix = random.nextInt(productCache.size());
}
Product product = productCache.get(productKey + randomSuffix);
if (!productIds.contains(product.id)) {
purchasesCache.put(customer.id, product.id);
productIds.add(product.id);
}
}
createProductAssociativityGraphPerCategory();
ProductRecommendationSystem recommendationSystem = new ProductRecommendationSystem(7);
for (int i = 0; i < activeProductSize * 2; i++) {
Customer customer = customerCache.get(customerKey + random.nextInt(customerCache.size()));
Product product = productCache.get(productKey + random.nextInt(activeProductSize));
List<Product> recommendations = recommendationSystem.purchase(customer, product);
System.out.printf("%s%n", recommendations);
}
}
public DocumentMapper documentMapperWithAutoCreate(String type) {
DocumentMapper mapper = mappers.get(type);
if (mapper != null) {
return mapper;
}
if (!dynamic) {
throw new TypeMissingException(
index, type, "trying to auto create mapping, but dynamic mapping is disabled");
}
// go ahead and dynamically create it
synchronized (typeMutex) {
mapper = mappers.get(type);
if (mapper != null) {
return mapper;
}
merge(type, null, true);
return mappers.get(type);
}
}
private ReferenceInfo.IndexType getColumnIndexType(Map<String, Object> columnProperties) {
String indexType = (String) columnProperties.get("index");
String analyzerName = (String) columnProperties.get("analyzer");
if (indexType != null) {
if (indexType.equals(ReferenceInfo.IndexType.NOT_ANALYZED.toString())) {
return ReferenceInfo.IndexType.NOT_ANALYZED;
} else if (indexType.equals(ReferenceInfo.IndexType.NO.toString())) {
return ReferenceInfo.IndexType.NO;
} else if (indexType.equals(ReferenceInfo.IndexType.ANALYZED.toString())
&& analyzerName != null
&& !analyzerName.equals("keyword")) {
return ReferenceInfo.IndexType.ANALYZED;
}
} // default indexType is analyzed so need to check analyzerName if indexType is null
else if (analyzerName != null && !analyzerName.equals("keyword")) {
return ReferenceInfo.IndexType.ANALYZED;
}
return ReferenceInfo.IndexType.NOT_ANALYZED;
}
public String urlTransform(String baseurl, Map<String, String> param) {
String url = baseurl;
for (String key : param.keySet()) {
url = url + "&" + key + "=" + param.get(key);
}
System.out.println(url);
return url;
}
@NotNull
private static List<Label> copyLabels(
Collection<Label> labels, Map<Label, Label> originalToCopy) {
List<Label> newLabels = Lists.newArrayList();
for (Label label : labels) {
Label newLabel = originalToCopy.get(label);
newLabels.add(newLabel != null ? newLabel : label);
}
return newLabels;
}
@NotNull
private ClassifierDescriptor modifyTypeClassifier(
@NotNull JetType autoType, @NotNull List<TypeAndVariance> typesFromSuper) {
ClassifierDescriptor classifier = autoType.getConstructor().getDeclarationDescriptor();
if (!(classifier instanceof ClassDescriptor)) {
assert classifier != null : "no declaration descriptor for type " + autoType;
if (classifier instanceof TypeParameterDescriptor
&& autoTypeParameterToModified.containsKey(classifier)) {
return autoTypeParameterToModified.get(classifier);
}
return classifier;
}
ClassDescriptor klass = (ClassDescriptor) classifier;
CollectionClassMapping collectionMapping = CollectionClassMapping.getInstance();
boolean someSupersMutable = false;
boolean someSupersCovariantReadOnly = false;
boolean someSupersNotCovariantReadOnly = false;
for (TypeAndVariance typeFromSuper : typesFromSuper) {
ClassifierDescriptor classifierFromSuper =
typeFromSuper.type.getConstructor().getDeclarationDescriptor();
if (classifierFromSuper instanceof ClassDescriptor) {
ClassDescriptor classFromSuper = (ClassDescriptor) classifierFromSuper;
if (collectionMapping.isMutableCollection(classFromSuper)) {
someSupersMutable = true;
} else if (collectionMapping.isReadOnlyCollection(classFromSuper)) {
if (typeFromSuper.varianceOfPosition == Variance.OUT_VARIANCE) {
someSupersCovariantReadOnly = true;
} else {
someSupersNotCovariantReadOnly = true;
}
}
}
}
if (someSupersMutable && someSupersNotCovariantReadOnly) {
reportError("Incompatible types in superclasses: " + typesFromSuper);
return classifier;
} else if (someSupersMutable) {
if (collectionMapping.isReadOnlyCollection(klass)) {
return collectionMapping.convertReadOnlyToMutable(klass);
}
} else if (someSupersNotCovariantReadOnly || someSupersCovariantReadOnly) {
if (collectionMapping.isMutableCollection(klass)) {
return collectionMapping.convertMutableToReadOnly(klass);
}
}
ClassifierDescriptor fixed =
PropagationHeuristics.tryToFixOverridingTWithRawType(this, typesFromSuper);
return fixed != null ? fixed : classifier;
}
/** Replace document refids with the actual references upon deserialization. */
private void documentIdToReference(Cluster cluster, Map<String, Document> documents) {
if (cluster.documentIds != null) {
for (Cluster.DocumentRefid documentRefid : cluster.documentIds) {
cluster.addDocuments(documents.get(documentRefid.refid));
}
}
for (Cluster subcluster : cluster.getSubclusters()) {
documentIdToReference(subcluster, documents);
}
}
public final Table getTable(String tableName, boolean caseSensitive) {
if (caseSensitive) {
return compositeTableMap.get(tableName);
} else {
final TableEntry tableEntry = tableMapInsensitive.get(tableName);
if (tableEntry != null) {
return tableEntry.getTable();
}
final FunctionEntry entry = nullaryFunctionMapInsensitive.get(tableName);
if (entry != null) {
return ((TableMacro) entry.getFunction()).apply(ImmutableList.of());
}
for (String name : schema.getTableNames()) {
if (name.equalsIgnoreCase(tableName)) {
return schema.getTable(name);
}
}
return null;
}
}
private ColumnIdent getCustomRoutingCol() {
if (defaultMappingMetaData != null) {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
String routingPath = (String) metaMap.get("routing");
if (routingPath != null && !routingPath.equals(ID)) {
return ColumnIdent.fromPath(routingPath);
}
}
}
return null;
}
/**
* Creates a {@link ProcessingResult} with the provided <code>attributes</code>. Assigns unique
* document identifiers if documents are present in the <code>attributes</code> map (under the key
* {@link AttributeNames#DOCUMENTS}).
*/
@SuppressWarnings("unchecked")
ProcessingResult(Map<String, Object> attributes) {
this.attributes = attributes;
// Replace a modifiable collection of documents with an unmodifiable one
final List<Document> documents = (List<Document>) attributes.get(AttributeNames.DOCUMENTS);
if (documents != null) {
Document.assignDocumentIds(documents);
attributes.put(AttributeNames.DOCUMENTS, Collections.unmodifiableList(documents));
}
// Replace a modifiable collection of clusters with an unmodifiable one
final List<Cluster> clusters = (List<Cluster>) attributes.get(AttributeNames.CLUSTERS);
if (clusters != null) {
Cluster.assignClusterIds(clusters);
attributes.put(AttributeNames.CLUSTERS, Collections.unmodifiableList(clusters));
}
// Store a reference to attributes as an unmodifiable map
this.attributesView = Collections.unmodifiableMap(attributes);
}