Ejemplo n.º 1
0
 private void processHiveEntity(
     HiveMetaStoreBridge dgiBridge,
     HiveEventContext event,
     Entity entity,
     Set<String> dataSetsProcessed,
     SortedMap<Entity, Referenceable> dataSets,
     Set<Referenceable> entities)
     throws Exception {
   if (entity.getType() == Type.TABLE || entity.getType() == Type.PARTITION) {
     final String tblQFName =
         dgiBridge.getTableQualifiedName(dgiBridge.getClusterName(), entity.getTable());
     if (!dataSetsProcessed.contains(tblQFName)) {
       LinkedHashMap<Type, Referenceable> result =
           createOrUpdateEntities(dgiBridge, event, entity, false);
       dataSets.put(entity, result.get(Type.TABLE));
       dataSetsProcessed.add(tblQFName);
       entities.addAll(result.values());
     }
   } else if (entity.getType() == Type.DFS_DIR) {
     final String pathUri = lower(new Path(entity.getLocation()).toString());
     LOG.info("Registering DFS Path {} ", pathUri);
     if (!dataSetsProcessed.contains(pathUri)) {
       Referenceable hdfsPath = dgiBridge.fillHDFSDataSet(pathUri);
       dataSets.put(entity, hdfsPath);
       dataSetsProcessed.add(pathUri);
       entities.add(hdfsPath);
     }
   }
 }
Ejemplo n.º 2
0
  private static void addDatasets(
      HiveOperation op, StringBuilder buffer, final Map<Entity, Referenceable> refs) {
    if (refs != null) {
      for (Entity input : refs.keySet()) {
        final Entity entity = input;

        // HiveOperation.QUERY type encompasses INSERT, INSERT_OVERWRITE, UPDATE, DELETE, PATH_WRITE
        // operations
        if (addQueryType(op, entity)) {
          buffer.append(SEP);
          buffer.append(((WriteEntity) entity).getWriteType().name());
        }
        if (Type.DFS_DIR.equals(entity.getType()) || Type.LOCAL_DIR.equals(entity.getType())) {
          LOG.debug(
              "Skipping dfs dir addition into process qualified name {} ",
              refs.get(input).get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME));
        } else {
          buffer.append(SEP);
          String dataSetQlfdName =
              (String) refs.get(input).get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
          // '/' breaks query parsing on ATLAS
          buffer.append(dataSetQlfdName.toLowerCase().replaceAll("/", ""));
        }
      }
    }
  }
Ejemplo n.º 3
0
 private Entity getEntityByType(Set<? extends Entity> entities, Type entityType) {
   for (Entity entity : entities) {
     if (entity.getType() == entityType) {
       return entity;
     }
   }
   return null;
 }
Ejemplo n.º 4
0
 private LinkedHashMap<Type, Referenceable> handleEventOutputs(
     HiveMetaStoreBridge dgiBridge, HiveEventContext event, Type entityType) throws Exception {
   for (Entity entity : event.getOutputs()) {
     if (entity.getType() == entityType) {
       return createOrUpdateEntities(dgiBridge, event, entity, true);
     }
   }
   return null;
 }
 // Build the hierarchy of authorizable object for the given entity type.
 private List<DBModelAuthorizable> getAuthzHierarchyFromEntity(Entity entity) {
   List<DBModelAuthorizable> objectHierarchy = new ArrayList<DBModelAuthorizable>();
   switch (entity.getType()) {
     case TABLE:
       objectHierarchy.add(new Database(entity.getTable().getDbName()));
       objectHierarchy.add(new Table(entity.getTable().getTableName()));
       break;
     case PARTITION:
     case DUMMYPARTITION:
       objectHierarchy.add(new Database(entity.getPartition().getTable().getDbName()));
       objectHierarchy.add(new Table(entity.getPartition().getTable().getTableName()));
       break;
     case DFS_DIR:
     case LOCAL_DIR:
       try {
         objectHierarchy.add(
             parseURI(entity.toString(), entity.getType().equals(Entity.Type.LOCAL_DIR)));
       } catch (Exception e) {
         throw new AuthorizationException("Failed to get File URI", e);
       }
       break;
     case DATABASE:
     case FUNCTION:
       // TODO use database entities from compiler instead of capturing from AST
       break;
     default:
       throw new UnsupportedOperationException(
           "Unsupported entity type " + entity.getType().name());
   }
   return objectHierarchy;
 }
Ejemplo n.º 6
0
  private void handleExternalTables(
      final HiveMetaStoreBridge dgiBridge,
      final HiveEventContext event,
      final LinkedHashMap<Type, Referenceable> tables)
      throws HiveException, MalformedURLException {
    List<Referenceable> entities = new ArrayList<>();
    final Entity hiveEntity = getEntityByType(event.getOutputs(), Type.TABLE);
    Table hiveTable = hiveEntity.getTable();
    // Refresh to get the correct location
    hiveTable = dgiBridge.hiveClient.getTable(hiveTable.getDbName(), hiveTable.getTableName());

    final String location = lower(hiveTable.getDataLocation().toString());
    if (hiveTable != null && TableType.EXTERNAL_TABLE.equals(hiveTable.getTableType())) {
      LOG.info("Registering external table process {} ", event.getQueryStr());
      final ReadEntity dfsEntity = new ReadEntity();
      dfsEntity.setTyp(Type.DFS_DIR);
      dfsEntity.setName(location);

      SortedMap<Entity, Referenceable> inputs =
          new TreeMap<Entity, Referenceable>(entityComparator) {
            {
              put(dfsEntity, dgiBridge.fillHDFSDataSet(location));
            }
          };

      SortedMap<Entity, Referenceable> outputs =
          new TreeMap<Entity, Referenceable>(entityComparator) {
            {
              put(hiveEntity, tables.get(Type.TABLE));
            }
          };

      Referenceable processReferenceable =
          getProcessReferenceable(dgiBridge, event, inputs, outputs);
      String tableQualifiedName =
          dgiBridge.getTableQualifiedName(dgiBridge.getClusterName(), hiveTable);

      if (isCreateOp(event)) {
        processReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName);
      }
      entities.addAll(tables.values());
      entities.add(processReferenceable);
      event.addMessage(new HookNotification.EntityUpdateRequest(event.getUser(), entities));
    }
  }
Ejemplo n.º 7
0
 private static boolean addQueryType(HiveOperation op, Entity entity) {
   if (WriteEntity.class.isAssignableFrom(entity.getClass())) {
     if (((WriteEntity) entity).getWriteType() != null && op.equals(HiveOperation.QUERY)) {
       switch (((WriteEntity) entity).getWriteType()) {
         case INSERT:
         case INSERT_OVERWRITE:
         case UPDATE:
         case DELETE:
         case PATH_WRITE:
           return true;
         default:
       }
     }
   }
   return false;
 }
 // Check if the given entity is identified as dummy by Hive compilers.
 private boolean isDummyEntity(Entity entity) {
   return entity.isDummy();
 }
Ejemplo n.º 9
0
  private LinkedHashMap<Type, Referenceable> createOrUpdateEntities(
      HiveMetaStoreBridge dgiBridge,
      HiveEventContext event,
      Entity entity,
      boolean skipTempTables,
      Table existTable)
      throws Exception {
    Database db = null;
    Table table = null;
    Partition partition = null;
    LinkedHashMap<Type, Referenceable> result = new LinkedHashMap<>();
    List<Referenceable> entities = new ArrayList<>();

    switch (entity.getType()) {
      case DATABASE:
        db = entity.getDatabase();
        break;

      case TABLE:
        table = entity.getTable();
        db = dgiBridge.hiveClient.getDatabase(table.getDbName());
        break;

      case PARTITION:
        partition = entity.getPartition();
        table = partition.getTable();
        db = dgiBridge.hiveClient.getDatabase(table.getDbName());
        break;
    }

    db = dgiBridge.hiveClient.getDatabase(db.getName());
    Referenceable dbEntity = dgiBridge.createDBInstance(db);

    entities.add(dbEntity);
    result.put(Type.DATABASE, dbEntity);

    Referenceable tableEntity = null;

    if (table != null) {
      if (existTable != null) {
        table = existTable;
      } else {
        table = dgiBridge.hiveClient.getTable(table.getDbName(), table.getTableName());
      }
      // If its an external table, even though the temp table skip flag is on,
      // we create the table since we need the HDFS path to temp table lineage.
      if (skipTempTables
          && table.isTemporary()
          && !TableType.EXTERNAL_TABLE.equals(table.getTableType())) {
        LOG.debug(
            "Skipping temporary table registration {} since it is not an external table {} ",
            table.getTableName(),
            table.getTableType().name());

      } else {
        tableEntity = dgiBridge.createTableInstance(dbEntity, table);
        entities.add(tableEntity);
        result.put(Type.TABLE, tableEntity);
      }
    }

    event.addMessage(new HookNotification.EntityUpdateRequest(event.getUser(), entities));
    return result;
  }
Ejemplo n.º 10
0
 @Override
 public int compare(Entity o1, Entity o2) {
   return o1.getName().toLowerCase().compareTo(o2.getName().toLowerCase());
 }