Example #1
0
  private static List<RelationshipInfo> reduceNodeTypes(
      final List<RelationshipInfo> sourceList, Map<String, TypeInfo> typeInfos) {

    final List<RelationshipInfo> reducedList = new ArrayList<>();
    final Set<String> startNodeTypes = new LinkedHashSet<>();
    final Set<String> endNodeTypes = new LinkedHashSet<>();
    String relType = null;

    for (final RelationshipInfo info : sourceList) {

      startNodeTypes.add(info.getStartNodeType());
      endNodeTypes.add(info.getEndNodeType());

      // set relType on first hit (should all be the same!)
      if (relType == null) {
        relType = info.getRelType();
      }
    }

    int startTypeCount = startNodeTypes.size();
    int endTypeCount = endNodeTypes.size();
    String commonStartType = null;
    String commonEndType = null;

    if (startTypeCount == 1) {

      commonStartType = startNodeTypes.iterator().next();

    } else {

      commonStartType = reduceTypeToCommonSupertype(startNodeTypes, typeInfos);
    }

    if (endTypeCount == 1) {

      commonEndType = endNodeTypes.iterator().next();

    } else {

      commonEndType = reduceTypeToCommonSupertype(endNodeTypes, typeInfos);
    }

    if (commonStartType != null && commonEndType != null) {

      reducedList.add(new RelationshipInfo(commonStartType, commonEndType, relType));
    }

    return reducedList;
  }
Example #2
0
  public static void analyzeSchema() {

    final App app = StructrApp.getInstance();
    final FileBasedHashLongMap<NodeInfo> nodeIdMap =
        new FileBasedHashLongMap<>(userHome + File.separator + ".structrSchemaAnalyzer");
    final GraphDatabaseService graphDb = app.command(GraphDatabaseCommand.class).execute();
    final ConfigurationProvider configuration = Services.getInstance().getConfigurationProvider();
    final Set<NodeInfo> nodeTypes = new LinkedHashSet<>();
    final Set<RelationshipInfo> relationships = new LinkedHashSet<>();
    final Map<String, SchemaNode> schemaNodes = new LinkedHashMap<>();
    final Map<String, List<TypeInfo>> typeInfoTypeMap = new LinkedHashMap<>();
    final List<TypeInfo> reducedTypeInfos = new LinkedList<>();
    final List<TypeInfo> typeInfos = new LinkedList<>();
    Iterator<Relationship> relIterator = null;
    Iterator<Node> nodeIterator = null;

    logger.log(Level.INFO, "Fetching all nodes iterator..");

    try (final Tx tx = app.tx()) {

      nodeIterator =
          Iterables.filter(
                  new StructrAndSpatialPredicate(false, false, true),
                  GlobalGraphOperations.at(graphDb).getAllNodes())
              .iterator();
      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Starting to analyze nodes..");

    NodeServiceCommand.bulkGraphOperation(
        SecurityContext.getSuperUserInstance(),
        nodeIterator,
        100000,
        "Analyzing nodes",
        new BulkGraphOperation<Node>() {

          @Override
          public void handleGraphObject(final SecurityContext securityContext, final Node node)
              throws FrameworkException {

            final NodeInfo nodeInfo = new NodeInfo(node);

            // hashcode of nodeInfo is derived from its property and type signature!
            nodeTypes.add(nodeInfo);

            // add node ID to our new test datastructure
            nodeIdMap.add(nodeInfo, node.getId());
          }
        });

    logger.log(Level.INFO, "Identifying common base classes..");

    try (final Tx tx = app.tx(true, false, false)) {

      // nodeTypes now contains all existing node types and their property sets
      identifyCommonBaseClasses(app, nodeTypes, nodeIdMap, typeInfos);

      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Collecting type information..");

    try (final Tx tx = app.tx(true, false, false)) {

      // group type infos by type
      collectTypeInfos(typeInfos, typeInfoTypeMap);

      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Aggregating type information..");

    try (final Tx tx = app.tx(true, false, false)) {

      // reduce type infos with more than one type
      reduceTypeInfos(typeInfoTypeMap, reducedTypeInfos);

      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Identifying property sets..");
    try (final Tx tx = app.tx(true, false, false)) {

      // intersect property sets of type infos
      intersectPropertySets(reducedTypeInfos);

      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Sorting result..");
    try (final Tx tx = app.tx(false, false, false)) {

      // sort type infos
      Collections.sort(reducedTypeInfos, new HierarchyComparator(false));

      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    final Map<String, TypeInfo> reducedTypeInfoMap = new LinkedHashMap<>();

    for (final TypeInfo info : reducedTypeInfos) {

      final String type = info.getPrimaryType();

      // map TypeInfo to type for later use
      reducedTypeInfoMap.put(type, info);

      logger.log(Level.INFO, "Starting with setting of type and ID for type {0}", type);

      NodeServiceCommand.bulkGraphOperation(
          SecurityContext.getSuperUserInstance(),
          info.getNodeIds().iterator(),
          10000,
          "Setting type and ID",
          new BulkGraphOperation<Long>() {

            @Override
            public void handleGraphObject(SecurityContext securityContext, Long nodeId)
                throws FrameworkException {

              final Node node = graphDb.getNodeById(nodeId);

              node.setProperty(GraphObject.id.dbName(), NodeServiceCommand.getNextUuid());
              node.setProperty(GraphObject.type.dbName(), type);
            }
          });
    }

    logger.log(Level.INFO, "Fetching all relationships iterator..");

    try (final Tx tx = app.tx(false, false, false)) {

      relIterator =
          Iterables.filter(
                  new StructrAndSpatialPredicate(false, false, true),
                  GlobalGraphOperations.at(graphDb).getAllRelationships())
              .iterator();
      tx.success();

    } catch (FrameworkException fex) {
      fex.printStackTrace();
    }

    logger.log(Level.INFO, "Starting with analyzing relationships..");

    NodeServiceCommand.bulkGraphOperation(
        SecurityContext.getSuperUserInstance(),
        relIterator,
        10000,
        "Analyzing relationships",
        new BulkGraphOperation<Relationship>() {

          @Override
          public void handleGraphObject(SecurityContext securityContext, Relationship rel)
              throws FrameworkException {

            final Node startNode = rel.getStartNode();
            final Node endNode = rel.getEndNode();

            // make sure node has been successfully identified above
            if (startNode.hasProperty("type") && endNode.hasProperty("type")) {

              final String relationshipType = rel.getType().name();
              final String startNodeType = (String) startNode.getProperty("type");
              final String endNodeType = (String) endNode.getProperty("type");

              relationships.add(new RelationshipInfo(startNodeType, endNodeType, relationshipType));

              // create combined type on imported relationship
              if (startNodeType != null && endNodeType != null) {

                final String combinedType =
                    getCombinedType(startNodeType, relationshipType, endNodeType);

                logger.log(
                    Level.FINE,
                    "Combined relationship type {0} found for rel type {1}, start node type {2}, end node type {3}",
                    new Object[] {combinedType, relationshipType, startNodeType, endNodeType});

                rel.setProperty(GraphObject.type.dbName(), combinedType);
              }

              // create ID on imported relationship
              rel.setProperty(GraphObject.id.dbName(), NodeServiceCommand.getNextUuid());
            }
          }
        });

    logger.log(Level.INFO, "Grouping relationships..");

    // group relationships by type
    final Map<String, List<RelationshipInfo>> relTypeInfoMap = new LinkedHashMap<>();
    for (final RelationshipInfo relInfo : relationships) {

      // final String relType         = relInfo.getRelType();
      final String combinedType =
          getCombinedType(
              relInfo.getStartNodeType(), relInfo.getRelType(), relInfo.getEndNodeType());
      List<RelationshipInfo> infos = relTypeInfoMap.get(combinedType);

      if (infos == null) {

        infos = new LinkedList<>();
        relTypeInfoMap.put(combinedType, infos);
      }

      infos.add(relInfo);
    }

    logger.log(Level.INFO, "Aggregating relationship information..");

    final List<RelationshipInfo> reducedRelationshipInfos = new ArrayList<>();
    if ("true"
        .equals(
            Services.getInstance()
                .getConfigurationValue("importer.inheritancedetection", "true"))) {

      // reduce relationship infos into one
      for (final List<RelationshipInfo> infos : relTypeInfoMap.values()) {

        reducedRelationshipInfos.addAll(reduceNodeTypes(infos, reducedTypeInfoMap));
      }

    } else {

      reducedRelationshipInfos.addAll(relationships);
    }

    logger.log(Level.INFO, "Starting with schema node creation..");

    NodeServiceCommand.bulkGraphOperation(
        SecurityContext.getSuperUserInstance(),
        reducedTypeInfos.iterator(),
        100000,
        "Creating schema nodes",
        new BulkGraphOperation<TypeInfo>() {

          @Override
          public void handleGraphObject(SecurityContext securityContext, TypeInfo typeInfo)
              throws FrameworkException {

            final String type = typeInfo.getPrimaryType();
            if (!"ReferenceNode".equals(type)) {

              final Map<String, Class> props = typeInfo.getPropertySet();
              final PropertyMap propertyMap = new PropertyMap();

              // add properties
              for (final Map.Entry<String, Class> propertyEntry : props.entrySet()) {

                final String propertyName = propertyEntry.getKey();
                final Class propertyType = propertyEntry.getValue();

                // handle array types differently
                String propertyTypeName = propertyType.getSimpleName();
                if (propertyType.isArray()) {

                  // remove "[]" from the end and append "Array" to match the appropriate parser
                  propertyTypeName =
                      propertyTypeName.substring(0, propertyTypeName.length() - 2).concat("Array");
                }

                propertyMap.put(new StringProperty("_".concat(propertyName)), propertyTypeName);
              }

              // set node type which is in "name" property
              propertyMap.put(AbstractNode.name, type);

              // check if there is an existing Structr entity with the same type
              // and make the dynamic class extend the existing class if yes.
              final Class existingType = configuration.getNodeEntityClass(type);
              if (existingType != null) {

                propertyMap.put(SchemaNode.extendsClass, existingType.getName());

              } else if (!typeInfo.getOtherTypes().isEmpty()) {

                // only the first supertype is supported
                propertyMap.put(
                    SchemaNode.extendsClass, typeInfo.getSuperclass(reducedTypeInfoMap));
              }

              final SchemaNode existingNode =
                  app.nodeQuery(SchemaNode.class).andName(type).getFirst();
              if (existingNode != null) {

                for (final Entry<PropertyKey, Object> entry : propertyMap.entrySet()) {

                  existingNode.setProperty(entry.getKey(), entry.getValue());
                }

                schemaNodes.put(type, existingNode);

              } else {

                // create schema node
                schemaNodes.put(type, app.create(SchemaNode.class, propertyMap));
              }
            }
          }
        });

    logger.log(Level.INFO, "Starting with schema relationship creation..");

    NodeServiceCommand.bulkGraphOperation(
        SecurityContext.getSuperUserInstance(),
        reducedRelationshipInfos.iterator(),
        100000,
        "Creating schema relationships",
        new BulkGraphOperation<RelationshipInfo>() {

          @Override
          public void handleGraphObject(SecurityContext securityContext, RelationshipInfo template)
              throws FrameworkException {

            final SchemaNode startNode = schemaNodes.get(template.getStartNodeType());
            final SchemaNode endNode = schemaNodes.get(template.getEndNodeType());
            final String relationshipType = template.getRelType();
            final PropertyMap propertyMap = new PropertyMap();

            propertyMap.put(SchemaRelationshipNode.sourceId, startNode.getUuid());
            propertyMap.put(SchemaRelationshipNode.targetId, endNode.getUuid());
            propertyMap.put(SchemaRelationshipNode.relationshipType, relationshipType);

            app.create(SchemaRelationshipNode.class, propertyMap);
          }
        });

    logger.log(Level.INFO, "Starting with index rebuild..");

    // rebuild index
    app.command(BulkRebuildIndexCommand.class).execute(Collections.EMPTY_MAP);
  }