コード例 #1
0
 private void dumpGraph() {
   AtlasGraph<?, ?> graph = TestUtils.getGraph();
   for (AtlasVertex<?, ?> v : graph.getVertices()) {
     System.out.println("****v = " + GraphHelper.vertexString(v));
     for (AtlasEdge<?, ?> e : v.getEdges(AtlasEdgeDirection.OUT)) {
       System.out.println("****e = " + GraphHelper.edgeString(e));
     }
   }
 }
コード例 #2
0
  public static void updateVertexPreUpdate(
      AtlasStructDef structDef,
      AtlasStructType structType,
      AtlasVertex vertex,
      AtlasTypeDefGraphStoreV1 typeDefStore)
      throws AtlasBaseException {

    List<String> attrNames = new ArrayList<>();
    if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
      for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
        attrNames.add(attributeDef.getName());
      }
    }

    List<String> currAttrNames =
        vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(structDef), List.class);

    // delete attributes that are not present in updated structDef
    if (CollectionUtils.isNotEmpty(currAttrNames)) {
      for (String currAttrName : currAttrNames) {
        if (!attrNames.contains(currAttrName)) {
          throw new AtlasBaseException(
              AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED, structDef.getName(), currAttrName);
        }
      }
    }

    typeDefStore.updateTypeVertex(structDef, vertex);

    // add/update attributes that are present in updated structDef
    if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
      for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
        if (CollectionUtils.isEmpty(currAttrNames)
            || !currAttrNames.contains(attributeDef.getName())) {
          // new attribute - only allow if optional
          if (!attributeDef.isOptional()) {
            throw new AtlasBaseException(
                AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE,
                structDef.getName(),
                attributeDef.getName());
          }
        }

        String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());

        AtlasGraphUtilsV1.setProperty(
            vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType));
      }
    }

    AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
  }
コード例 #3
0
  @Override
  public AtlasStructDef getByName(String name) throws AtlasBaseException {
    if (LOG.isDebugEnabled()) {
      LOG.debug("==> AtlasStructDefStoreV1.getByName({})", name);
    }

    AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);

    if (vertex == null) {
      throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
    }

    vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, String.class);

    AtlasStructDef ret = toStructDef(vertex);

    if (LOG.isDebugEnabled()) {
      LOG.debug("<== AtlasStructDefStoreV1.getByName({}): {}", name, ret);
    }

    return ret;
  }
コード例 #4
0
  public static AtlasStructDef toStructDef(
      AtlasVertex vertex, AtlasStructDef structDef, AtlasTypeDefGraphStoreV1 typeDefStore)
      throws AtlasBaseException {
    AtlasStructDef ret = (structDef != null) ? structDef : new AtlasStructDef();

    typeDefStore.vertexToTypeDef(vertex, ret);

    List<AtlasAttributeDef> attributeDefs = new ArrayList<>();
    List<String> attrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);

    if (CollectionUtils.isNotEmpty(attrNames)) {
      for (String attrName : attrNames) {
        String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName);
        String attribJson = vertex.getProperty(propertyKey, String.class);

        attributeDefs.add(
            toAttributeDefFromJson(
                structDef, AtlasType.fromJson(attribJson, Map.class), typeDefStore));
      }
    }
    ret.setAttributeDefs(attributeDefs);

    return ret;
  }
コード例 #5
0
  protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
    // Update external references(incoming edges) to this vertex
    LOG.debug(
        "Setting the external references to {} to null(removing edges)", string(instanceVertex));
    Iterator<AtlasEdge> edges = instanceVertex.getEdges(AtlasEdgeDirection.IN).iterator();

    while (edges.hasNext()) {
      AtlasEdge edge = edges.next();
      Id.EntityState edgeState = GraphHelper.getState(edge);
      if (edgeState == Id.EntityState.ACTIVE) {
        // Delete only the active edge references
        AttributeInfo attribute = getAttributeForEdge(edge.getLabel());
        // TODO use delete edge instead??
        deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute.name);
      }
    }
    _deleteVertex(instanceVertex, force);
  }
コード例 #6
0
  private static void addReferencesForAttribute(
      AtlasVertex vertex, AtlasAttributeDef attributeDef, AtlasTypeDefGraphStoreV1 typeDefStore)
      throws AtlasBaseException {
    Set<String> referencedTypeNames =
        AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName());

    String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY, String.class);

    for (String referencedTypeName : referencedTypeNames) {
      if (!AtlasTypeUtil.isBuiltInType(referencedTypeName)) {
        AtlasVertex referencedTypeVertex = typeDefStore.findTypeVertexByName(referencedTypeName);

        if (referencedTypeVertex == null) {
          throw new AtlasBaseException(
              AtlasErrorCode.UNKNOWN_TYPE, referencedTypeName, typeName, attributeDef.getName());
        }

        String label = AtlasGraphUtilsV1.getEdgeLabel(typeName, attributeDef.getName());

        typeDefStore.getOrCreateEdge(vertex, referencedTypeVertex, label);
      }
    }
  }
コード例 #7
0
  /**
   * Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of
   * outVertex
   *
   * @param outVertex
   * @param inVertex
   * @param attributeName
   * @throws AtlasException
   */
  protected void deleteEdgeBetweenVertices(
      AtlasVertex outVertex, AtlasVertex inVertex, String attributeName) throws AtlasException {
    LOG.debug(
        "Removing edge from {} to {} with attribute name {}",
        string(outVertex),
        string(inVertex),
        attributeName);
    String typeName = GraphHelper.getTypeName(outVertex);
    String outId = GraphHelper.getIdFromVertex(outVertex);
    Id.EntityState state = GraphHelper.getState(outVertex);
    if ((outId != null && RequestContext.get().isDeletedEntity(outId))
        || state == Id.EntityState.DELETED) {
      // If the reference vertex is marked for deletion, skip updating the reference
      return;
    }

    IDataType type = typeSystem.getDataType(IDataType.class, typeName);
    AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName);
    String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName);
    String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
    AtlasEdge edge = null;

    switch (attributeInfo.dataType().getTypeCategory()) {
      case CLASS:
        // If its class attribute, its the only edge between two vertices
        if (attributeInfo.multiplicity.nullAllowed()) {
          edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
          if (shouldUpdateReverseAttribute) {
            GraphHelper.setProperty(outVertex, propertyName, null);
          }
        } else {
          // Cannot unset a required attribute.
          throw new NullRequiredAttributeException(
              "Cannot unset required attribute "
                  + GraphHelper.getQualifiedFieldName(type, attributeName)
                  + " on "
                  + GraphHelper.getVertexDetails(outVertex)
                  + " edge = "
                  + edgeLabel);
        }
        break;

      case ARRAY:
        // If its array attribute, find the right edge between the two vertices and update array
        // property
        List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
        if (elements != null) {
          elements =
              new ArrayList<>(
                  elements); // Make a copy, else list.remove reflects on titan.getProperty()
          for (String elementEdgeId : elements) {
            AtlasEdge elementEdge =
                graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
            if (elementEdge == null) {
              continue;
            }

            AtlasVertex elementVertex = elementEdge.getInVertex();
            if (elementVertex.equals(inVertex)) {
              edge = elementEdge;

              // TODO element.size includes deleted items as well. should exclude
              if (!attributeInfo.multiplicity.nullAllowed()
                  && elements.size() <= attributeInfo.multiplicity.lower) {
                // Deleting this edge would violate the attribute's lower bound.
                throw new NullRequiredAttributeException(
                    "Cannot remove array element from required attribute "
                        + GraphHelper.getQualifiedFieldName(type, attributeName)
                        + " on "
                        + GraphHelper.getVertexDetails(outVertex)
                        + " "
                        + GraphHelper.getEdgeDetails(elementEdge));
              }

              if (shouldUpdateReverseAttribute) {
                // if composite attribute, remove the reference as well. else, just remove the edge
                // for example, when table is deleted, process still references the table
                // but when column is deleted, table will not reference the deleted column
                LOG.debug(
                    "Removing edge {} from the array attribute {}",
                    string(elementEdge),
                    attributeName);
                elements.remove(elementEdge.getId().toString());
                GraphHelper.setProperty(outVertex, propertyName, elements);
                break;
              }
            }
          }
        }
        break;

      case MAP:
        // If its map attribute, find the right edge between two vertices and update map property
        List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
        if (keys != null) {
          keys =
              new ArrayList<>(
                  keys); // Make a copy, else list.remove reflects on titan.getProperty()
          for (String key : keys) {
            String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
            String mapEdgeId =
                GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
            AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
            if (mapEdge != null) {
              AtlasVertex mapVertex = mapEdge.getInVertex();
              if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
                // TODO keys.size includes deleted items as well. should exclude
                if (attributeInfo.multiplicity.nullAllowed()
                    || keys.size() > attributeInfo.multiplicity.lower) {
                  edge = mapEdge;
                } else {
                  // Deleting this entry would violate the attribute's lower bound.
                  throw new NullRequiredAttributeException(
                      "Cannot remove map entry "
                          + keyPropertyName
                          + " from required attribute "
                          + GraphHelper.getQualifiedFieldName(type, attributeName)
                          + " on "
                          + GraphHelper.getVertexDetails(outVertex)
                          + " "
                          + GraphHelper.getEdgeDetails(mapEdge));
                }

                if (shouldUpdateReverseAttribute) {
                  // remove this key
                  LOG.debug(
                      "Removing edge {}, key {} from the map attribute {}",
                      string(mapEdge),
                      key,
                      attributeName);
                  keys.remove(key);
                  GraphHelper.setProperty(outVertex, propertyName, keys);
                  GraphHelper.setProperty(outVertex, keyPropertyName, null);
                }
                break;
              }
            }
          }
        }
        break;

      case STRUCT:
      case TRAIT:
        break;

      default:
        throw new IllegalStateException(
            "There can't be an edge from "
                + GraphHelper.getVertexDetails(outVertex)
                + " to "
                + GraphHelper.getVertexDetails(inVertex)
                + " with attribute name "
                + attributeName
                + " which is not class/array/map attribute");
    }

    if (edge != null) {
      deleteEdge(edge, false);
      RequestContext requestContext = RequestContext.get();
      GraphHelper.setProperty(
          outVertex,
          Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
          requestContext.getRequestTime());
      requestContext.recordEntityUpdate(outId);
    }
  }
コード例 #8
0
  private static AtlasAttributeDef toAttributeDefFromJson(
      AtlasStructDef structDef, Map attribInfo, AtlasTypeDefGraphStoreV1 typeDefStore)
      throws AtlasBaseException {
    AtlasAttributeDef ret = new AtlasAttributeDef();

    ret.setName((String) attribInfo.get("name"));
    ret.setTypeName((String) attribInfo.get("dataType"));
    ret.setUnique((Boolean) attribInfo.get("isUnique"));
    ret.setIndexable((Boolean) attribInfo.get("isIndexable"));

    String attrTypeName = ret.getTypeName();

    if (AtlasTypeUtil.isArrayType(attrTypeName)) {
      Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(ret.getTypeName());

      if (typeNames.size() > 0) {
        attrTypeName = typeNames.iterator().next();
      }
    }

    if (!AtlasTypeUtil.isBuiltInType(attrTypeName)) {
      AtlasVertex attributeType = typeDefStore.findTypeVertexByName(attrTypeName);

      // check for isComposite/reverseAttributeName for entity types
      if (attributeType != null && typeDefStore.isTypeVertex(attributeType, TypeCategory.CLASS)) {
        String reverseAttribName = (String) attribInfo.get("reverseAttributeName");
        Boolean isComposite = (Boolean) attribInfo.get("isComposite");

        if (StringUtils.isNotBlank(reverseAttribName) || isComposite) {
          if (AtlasTypeUtil.isMapType(attrTypeName)) {
            throw new AtlasBaseException(
                AtlasErrorCode.CONSTRAINT_NOT_SUPPORTED_ON_MAP_TYPE,
                structDef.getName(),
                ret.getName(),
                attrTypeName);
          }

          String refAttributeName = null;
          List<String> attrNames =
              attributeType.getProperty(AtlasGraphUtilsV1.getPropertyKey(attrTypeName), List.class);

          if (CollectionUtils.isNotEmpty(attrNames)) {
            for (String attrName : attrNames) {
              String attribJson =
                  attributeType.getProperty(
                      AtlasGraphUtilsV1.getPropertyKey(attrTypeName, attrName), String.class);

              Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class);
              String refAttribType = (String) refAttrInfo.get("dataType");
              String refAttribRevAttribName = (String) refAttrInfo.get("reverseAttributeName");

              if (StringUtils.equals(refAttribType, structDef.getName())
                  && StringUtils.equals(refAttribRevAttribName, ret.getName())) {
                refAttributeName = (String) refAttrInfo.get("name");

                break;
              }
            }
          }

          if (isComposite) {
            if (StringUtils.isNotBlank(
                refAttributeName)) { // ex: hive_table.columns, hive_column.table
              Map<String, Object> params = new HashMap<>();
              params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttributeName);

              ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params));
            } else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference
              ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY));
            }
          }

          if (StringUtils.isNotBlank(reverseAttribName)) { // ex: hive_column.table
            Map<String, Object> params = new HashMap<>();
            params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE);

            ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params));
          }
        }
      }
    }

    Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
    Number minCount = (Number) multiplicity.get("lower");
    Number maxCount = (Number) multiplicity.get("upper");
    Boolean isUnique = (Boolean) multiplicity.get("isUnique");

    if (minCount == null || minCount.intValue() == 0) {
      ret.setOptional(true);
      ret.setValuesMinCount(0);
    } else {
      ret.setOptional(false);
      ret.setValuesMinCount(minCount.intValue());
    }

    if (maxCount == null || maxCount.intValue() < 2) {
      ret.setCardinality(AtlasAttributeDef.Cardinality.SINGLE);
      ret.setValuesMaxCount(1);
    } else {
      if (isUnique == null || isUnique == Boolean.FALSE) {
        ret.setCardinality(AtlasAttributeDef.Cardinality.LIST);
      } else {
        ret.setCardinality(AtlasAttributeDef.Cardinality.SET);
      }

      ret.setValuesMaxCount(maxCount.intValue());
    }

    return ret;
  }