private static void intersectPropertySets(final List<TypeInfo> reducedTypeInfos) {
// substract property set from type info with more than one
// occurrence from type infos with the given superclass
for (final TypeInfo info : reducedTypeInfos) {
if (info.getHierarchyLevel() > 1) {
final Set<String> supertypeKeySet = info.getPropertySet().keySet();
for (final TypeInfo subType : reducedTypeInfos) {
final Set<String> subtypeKeySet = subType.getPropertySet().keySet();
// only substract property set if it is a true subtype (and not the same :))
// if ( subType.getUsages() == 1 && subType.hasSuperclass(info.getPrimaryType())) {
if (subType.getHierarchyLevel() < info.getHierarchyLevel()
&& subType.hasSuperclass(info.getPrimaryType())) {
subtypeKeySet.removeAll(supertypeKeySet);
}
}
}
}
}
private static void collectTypeInfos(
final List<TypeInfo> typeInfos, final Map<String, List<TypeInfo>> typeInfoTypeMap) {
// collect type infos by type (to detect multiples)
for (final TypeInfo info : typeInfos) {
final String type = info.getPrimaryType();
List<TypeInfo> typeInfo = typeInfoTypeMap.get(type);
if (typeInfo == null) {
typeInfo = new LinkedList<>();
typeInfoTypeMap.put(type, typeInfo);
}
typeInfo.add(info);
}
}
public static void analyzeSchema() {
final App app = StructrApp.getInstance();
final FileBasedHashLongMap<NodeInfo> nodeIdMap =
new FileBasedHashLongMap<>(userHome + File.separator + ".structrSchemaAnalyzer");
final GraphDatabaseService graphDb = app.command(GraphDatabaseCommand.class).execute();
final ConfigurationProvider configuration = Services.getInstance().getConfigurationProvider();
final Set<NodeInfo> nodeTypes = new LinkedHashSet<>();
final Set<RelationshipInfo> relationships = new LinkedHashSet<>();
final Map<String, SchemaNode> schemaNodes = new LinkedHashMap<>();
final Map<String, List<TypeInfo>> typeInfoTypeMap = new LinkedHashMap<>();
final List<TypeInfo> reducedTypeInfos = new LinkedList<>();
final List<TypeInfo> typeInfos = new LinkedList<>();
Iterator<Relationship> relIterator = null;
Iterator<Node> nodeIterator = null;
logger.log(Level.INFO, "Fetching all nodes iterator..");
try (final Tx tx = app.tx()) {
nodeIterator =
Iterables.filter(
new StructrAndSpatialPredicate(false, false, true),
GlobalGraphOperations.at(graphDb).getAllNodes())
.iterator();
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Starting to analyze nodes..");
NodeServiceCommand.bulkGraphOperation(
SecurityContext.getSuperUserInstance(),
nodeIterator,
100000,
"Analyzing nodes",
new BulkGraphOperation<Node>() {
@Override
public void handleGraphObject(final SecurityContext securityContext, final Node node)
throws FrameworkException {
final NodeInfo nodeInfo = new NodeInfo(node);
// hashcode of nodeInfo is derived from its property and type signature!
nodeTypes.add(nodeInfo);
// add node ID to our new test datastructure
nodeIdMap.add(nodeInfo, node.getId());
}
});
logger.log(Level.INFO, "Identifying common base classes..");
try (final Tx tx = app.tx(true, false, false)) {
// nodeTypes now contains all existing node types and their property sets
identifyCommonBaseClasses(app, nodeTypes, nodeIdMap, typeInfos);
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Collecting type information..");
try (final Tx tx = app.tx(true, false, false)) {
// group type infos by type
collectTypeInfos(typeInfos, typeInfoTypeMap);
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Aggregating type information..");
try (final Tx tx = app.tx(true, false, false)) {
// reduce type infos with more than one type
reduceTypeInfos(typeInfoTypeMap, reducedTypeInfos);
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Identifying property sets..");
try (final Tx tx = app.tx(true, false, false)) {
// intersect property sets of type infos
intersectPropertySets(reducedTypeInfos);
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Sorting result..");
try (final Tx tx = app.tx(false, false, false)) {
// sort type infos
Collections.sort(reducedTypeInfos, new HierarchyComparator(false));
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
final Map<String, TypeInfo> reducedTypeInfoMap = new LinkedHashMap<>();
for (final TypeInfo info : reducedTypeInfos) {
final String type = info.getPrimaryType();
// map TypeInfo to type for later use
reducedTypeInfoMap.put(type, info);
logger.log(Level.INFO, "Starting with setting of type and ID for type {0}", type);
NodeServiceCommand.bulkGraphOperation(
SecurityContext.getSuperUserInstance(),
info.getNodeIds().iterator(),
10000,
"Setting type and ID",
new BulkGraphOperation<Long>() {
@Override
public void handleGraphObject(SecurityContext securityContext, Long nodeId)
throws FrameworkException {
final Node node = graphDb.getNodeById(nodeId);
node.setProperty(GraphObject.id.dbName(), NodeServiceCommand.getNextUuid());
node.setProperty(GraphObject.type.dbName(), type);
}
});
}
logger.log(Level.INFO, "Fetching all relationships iterator..");
try (final Tx tx = app.tx(false, false, false)) {
relIterator =
Iterables.filter(
new StructrAndSpatialPredicate(false, false, true),
GlobalGraphOperations.at(graphDb).getAllRelationships())
.iterator();
tx.success();
} catch (FrameworkException fex) {
fex.printStackTrace();
}
logger.log(Level.INFO, "Starting with analyzing relationships..");
NodeServiceCommand.bulkGraphOperation(
SecurityContext.getSuperUserInstance(),
relIterator,
10000,
"Analyzing relationships",
new BulkGraphOperation<Relationship>() {
@Override
public void handleGraphObject(SecurityContext securityContext, Relationship rel)
throws FrameworkException {
final Node startNode = rel.getStartNode();
final Node endNode = rel.getEndNode();
// make sure node has been successfully identified above
if (startNode.hasProperty("type") && endNode.hasProperty("type")) {
final String relationshipType = rel.getType().name();
final String startNodeType = (String) startNode.getProperty("type");
final String endNodeType = (String) endNode.getProperty("type");
relationships.add(new RelationshipInfo(startNodeType, endNodeType, relationshipType));
// create combined type on imported relationship
if (startNodeType != null && endNodeType != null) {
final String combinedType =
getCombinedType(startNodeType, relationshipType, endNodeType);
logger.log(
Level.FINE,
"Combined relationship type {0} found for rel type {1}, start node type {2}, end node type {3}",
new Object[] {combinedType, relationshipType, startNodeType, endNodeType});
rel.setProperty(GraphObject.type.dbName(), combinedType);
}
// create ID on imported relationship
rel.setProperty(GraphObject.id.dbName(), NodeServiceCommand.getNextUuid());
}
}
});
logger.log(Level.INFO, "Grouping relationships..");
// group relationships by type
final Map<String, List<RelationshipInfo>> relTypeInfoMap = new LinkedHashMap<>();
for (final RelationshipInfo relInfo : relationships) {
// final String relType = relInfo.getRelType();
final String combinedType =
getCombinedType(
relInfo.getStartNodeType(), relInfo.getRelType(), relInfo.getEndNodeType());
List<RelationshipInfo> infos = relTypeInfoMap.get(combinedType);
if (infos == null) {
infos = new LinkedList<>();
relTypeInfoMap.put(combinedType, infos);
}
infos.add(relInfo);
}
logger.log(Level.INFO, "Aggregating relationship information..");
final List<RelationshipInfo> reducedRelationshipInfos = new ArrayList<>();
if ("true"
.equals(
Services.getInstance()
.getConfigurationValue("importer.inheritancedetection", "true"))) {
// reduce relationship infos into one
for (final List<RelationshipInfo> infos : relTypeInfoMap.values()) {
reducedRelationshipInfos.addAll(reduceNodeTypes(infos, reducedTypeInfoMap));
}
} else {
reducedRelationshipInfos.addAll(relationships);
}
logger.log(Level.INFO, "Starting with schema node creation..");
NodeServiceCommand.bulkGraphOperation(
SecurityContext.getSuperUserInstance(),
reducedTypeInfos.iterator(),
100000,
"Creating schema nodes",
new BulkGraphOperation<TypeInfo>() {
@Override
public void handleGraphObject(SecurityContext securityContext, TypeInfo typeInfo)
throws FrameworkException {
final String type = typeInfo.getPrimaryType();
if (!"ReferenceNode".equals(type)) {
final Map<String, Class> props = typeInfo.getPropertySet();
final PropertyMap propertyMap = new PropertyMap();
// add properties
for (final Map.Entry<String, Class> propertyEntry : props.entrySet()) {
final String propertyName = propertyEntry.getKey();
final Class propertyType = propertyEntry.getValue();
// handle array types differently
String propertyTypeName = propertyType.getSimpleName();
if (propertyType.isArray()) {
// remove "[]" from the end and append "Array" to match the appropriate parser
propertyTypeName =
propertyTypeName.substring(0, propertyTypeName.length() - 2).concat("Array");
}
propertyMap.put(new StringProperty("_".concat(propertyName)), propertyTypeName);
}
// set node type which is in "name" property
propertyMap.put(AbstractNode.name, type);
// check if there is an existing Structr entity with the same type
// and make the dynamic class extend the existing class if yes.
final Class existingType = configuration.getNodeEntityClass(type);
if (existingType != null) {
propertyMap.put(SchemaNode.extendsClass, existingType.getName());
} else if (!typeInfo.getOtherTypes().isEmpty()) {
// only the first supertype is supported
propertyMap.put(
SchemaNode.extendsClass, typeInfo.getSuperclass(reducedTypeInfoMap));
}
final SchemaNode existingNode =
app.nodeQuery(SchemaNode.class).andName(type).getFirst();
if (existingNode != null) {
for (final Entry<PropertyKey, Object> entry : propertyMap.entrySet()) {
existingNode.setProperty(entry.getKey(), entry.getValue());
}
schemaNodes.put(type, existingNode);
} else {
// create schema node
schemaNodes.put(type, app.create(SchemaNode.class, propertyMap));
}
}
}
});
logger.log(Level.INFO, "Starting with schema relationship creation..");
NodeServiceCommand.bulkGraphOperation(
SecurityContext.getSuperUserInstance(),
reducedRelationshipInfos.iterator(),
100000,
"Creating schema relationships",
new BulkGraphOperation<RelationshipInfo>() {
@Override
public void handleGraphObject(SecurityContext securityContext, RelationshipInfo template)
throws FrameworkException {
final SchemaNode startNode = schemaNodes.get(template.getStartNodeType());
final SchemaNode endNode = schemaNodes.get(template.getEndNodeType());
final String relationshipType = template.getRelType();
final PropertyMap propertyMap = new PropertyMap();
propertyMap.put(SchemaRelationshipNode.sourceId, startNode.getUuid());
propertyMap.put(SchemaRelationshipNode.targetId, endNode.getUuid());
propertyMap.put(SchemaRelationshipNode.relationshipType, relationshipType);
app.create(SchemaRelationshipNode.class, propertyMap);
}
});
logger.log(Level.INFO, "Starting with index rebuild..");
// rebuild index
app.command(BulkRebuildIndexCommand.class).execute(Collections.EMPTY_MAP);
}
