Exemple #1
0
public class ChangeEmail extends BaseRequestHandler {
  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ChangeEmail.class);
  private static final String EMAIL_PARAMETER_NAME = "email";

  @Inject
  public ChangeEmail(
      UserRepository userRepository,
      WorkspaceRepository workspaceRepository,
      Configuration configuration) {
    super(userRepository, workspaceRepository, configuration);
  }

  @Override
  public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
      throws Exception {
    User user = getUser(request);
    String email = getRequiredParameter(request, EMAIL_PARAMETER_NAME);

    if (user != null) {
      if (email.length() > 0) {
        getUserRepository().setEmailAddress(user, email);
        LOGGER.info("changed email for user: %s", user.getUsername());
        respondWithSuccessJson(response);
      } else {
        respondWithBadRequest(response, EMAIL_PARAMETER_NAME, "new email may not be blank");
      }
    } else {
      LOGGER.error("current user not found while attempting to change email");
      respondWithAccessDenied(response, "current user not found");
    }
  }
}
  @Override
  public void contextInitialized(ServletContextEvent sce) {
    final ServletContext context = sce.getServletContext();

    LOGGER = LumifyLoggerFactory.getLogger(ApplicationBootstrap.class);

    LOGGER.info("Servlet context initialized...");
    try {
      if (context != null) {
        final Configuration config = fetchApplicationConfiguration(context);
        LOGGER.info("Running application with configuration:\n%s", config);

        InjectHelper.inject(this, LumifyBootstrap.bootstrapModuleMaker(config));

        // Store the injector in the context for a servlet to access later
        context.setAttribute(Injector.class.getName(), InjectHelper.getInjector());
        if (!config.get(Configuration.MODEL_PROVIDER).equals(Configuration.UNKNOWN_STRING)) {
          FrameworkUtils.initializeFramework(
              InjectHelper.getInjector(), userRepository.getSystemUser());
        }

        InjectHelper.getInjector().getInstance(OntologyRepository.class);

        LOGGER.warn(
            "JavaScript / Less modifications will not be reflected on server. Run `grunt watch` from webapp directory in development");
      } else {
        LOGGER.error("Servlet context could not be acquired!");
      }
    } catch (Exception ex) {
      LOGGER.error("Failed to initialize context", ex);
      throw new RuntimeException("Failed to initialize context", ex);
    }
  }
public class HibernateSessionManager {
  private static LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(HibernateSessionManager.class);
  private SessionFactory sessionFactory;

  private ThreadLocal<Session> threadLocalSession =
      new InheritableThreadLocal<Session>() {
        @Override
        protected Session initialValue() {
          LOGGER.info("Opening Hibernate session");
          return sessionFactory.openSession();
        }
      };

  // this is to prevent opening a session just to clear it
  private ThreadLocal<Boolean> threadLocalSessionIsOpen =
      new InheritableThreadLocal<Boolean>() {
        @Override
        protected Boolean initialValue() {
          return Boolean.FALSE;
        }
      };

  public HibernateSessionManager(SessionFactory factory) {
    sessionFactory = factory;
    threadLocalSession.remove();
  }

  public Session getSession() {
    threadLocalSessionIsOpen.set(Boolean.TRUE);
    return threadLocalSession.get();
  }

  public void clearSession() {
    if (threadLocalSessionIsOpen.get()) {
      Session session = threadLocalSession.get();
      if (session.isOpen()) { // double checking
        session.close();
        LOGGER.info("Closed Hibernate session");
      }
    }

    threadLocalSession.remove();
    threadLocalSessionIsOpen.remove();
  }
}
public class FFprobeMakeAndModelUtil {
  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(FFprobeMakeAndModelUtil.class);

  public static String getMake(JSONObject json) {
    return extractMakeAndModelFromJSON(json, "make", "make-eng");
  }

  public static String getModel(JSONObject json) {
    return extractMakeAndModelFromJSON(json, "model", "model-eng");
  }

  private static String extractMakeAndModelFromJSON(
      JSONObject json, String firstSearchString, String secondSearchString) {
    if (json == null) {
      return null;
    }

    JSONObject formatObject = json.optJSONObject("format");
    if (formatObject != null) {
      JSONObject tagsObject = formatObject.optJSONObject("tags");
      if (tagsObject != null) {
        String firstDesiredValue = tagsObject.optString(firstSearchString);
        if (!firstDesiredValue.equals("")) {
          return firstDesiredValue;
        }
        String secondDesiredValue = tagsObject.optString(secondSearchString);
        if (!secondDesiredValue.equals("")) {
          return secondDesiredValue;
        }
      }
    }

    LOGGER.debug(
        "Could not extract " + firstSearchString + " or " + secondSearchString + " from json.");
    return null;
  }
}
public class ResolveTermEntity extends BaseRequestHandler {
  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ResolveTermEntity.class);
  private static final String MULTI_VALUE_KEY = ResolveTermEntity.class.getName();
  private final Graph graph;
  private final AuditRepository auditRepository;
  private final OntologyRepository ontologyRepository;
  private final VisibilityTranslator visibilityTranslator;
  private final TermMentionRepository termMentionRepository;
  private final WorkspaceRepository workspaceRepository;
  private final WorkQueueRepository workQueueRepository;
  private final String artifactHasEntityIri;

  @Inject
  public ResolveTermEntity(
      final Graph graphRepository,
      final AuditRepository auditRepository,
      final OntologyRepository ontologyRepository,
      final UserRepository userRepository,
      final VisibilityTranslator visibilityTranslator,
      final Configuration configuration,
      final TermMentionRepository termMentionRepository,
      final WorkspaceRepository workspaceRepository,
      final WorkQueueRepository workQueueRepository) {
    super(userRepository, workspaceRepository, configuration);
    this.graph = graphRepository;
    this.auditRepository = auditRepository;
    this.ontologyRepository = ontologyRepository;
    this.visibilityTranslator = visibilityTranslator;
    this.termMentionRepository = termMentionRepository;
    this.workspaceRepository = workspaceRepository;
    this.workQueueRepository = workQueueRepository;

    this.artifactHasEntityIri =
        this.getConfiguration().get(Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
    if (this.artifactHasEntityIri == null) {
      throw new LumifyException(
          "Could not find configuration for " + Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
    }
  }

  @Override
  public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
      throws Exception {
    final String artifactId = getRequiredParameter(request, "artifactId");
    final String propertyKey = getRequiredParameter(request, "propertyKey");
    final long mentionStart = getRequiredParameterAsLong(request, "mentionStart");
    final long mentionEnd = getRequiredParameterAsLong(request, "mentionEnd");
    final String title = getRequiredParameter(request, "sign");
    final String conceptId = getRequiredParameter(request, "conceptId");
    final String visibilitySource = getRequiredParameter(request, "visibilitySource");
    final String resolvedVertexId = getOptionalParameter(request, "resolvedVertexId");
    final String justificationText = getOptionalParameter(request, "justificationText");
    final String sourceInfo = getOptionalParameter(request, "sourceInfo");
    String termMentionRowKeyString = getOptionalParameter(request, "rowKey");

    User user = getUser(request);
    String workspaceId = getActiveWorkspaceId(request);
    Workspace workspace = workspaceRepository.findById(workspaceId, user);

    Authorizations authorizations = getAuthorizations(request, user);

    JSONObject visibilityJson =
        GraphUtil.updateVisibilitySourceAndAddWorkspaceId(null, visibilitySource, workspaceId);
    LumifyVisibility visibility = this.visibilityTranslator.toVisibility(visibilityJson);
    if (!graph.isVisibilityValid(visibility.getVisibility(), authorizations)) {
      LOGGER.warn(
          "%s is not a valid visibility for %s user", visibilitySource, user.getDisplayName());
      respondWithBadRequest(response, "visibilitySource", STRINGS.getString("visibility.invalid"));
      chain.next(request, response);
      return;
    }

    Object id = resolvedVertexId == null ? graph.getIdGenerator().nextId() : resolvedVertexId;

    Concept concept = ontologyRepository.getConceptByIRI(conceptId);

    final Vertex artifactVertex = graph.getVertex(artifactId, authorizations);
    LumifyVisibility lumifyVisibility = visibilityTranslator.toVisibility(visibilityJson);
    Map<String, Object> metadata = new HashMap<String, Object>();
    LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setMetadata(metadata, visibilityJson);
    ElementMutation<Vertex> vertexMutation;
    Vertex vertex;
    if (resolvedVertexId != null) {
      vertex = graph.getVertex(id, authorizations);
      vertexMutation = vertex.prepareMutation();
    } else {
      vertexMutation = graph.prepareVertex(id, lumifyVisibility.getVisibility());
      GraphUtil.addJustificationToMutation(
          vertexMutation, justificationText, sourceInfo, lumifyVisibility);

      CONCEPT_TYPE.setProperty(
          vertexMutation, conceptId, metadata, lumifyVisibility.getVisibility());
      TITLE.addPropertyValue(
          vertexMutation, MULTI_VALUE_KEY, title, metadata, lumifyVisibility.getVisibility());

      vertex = vertexMutation.save(authorizations);

      auditRepository.auditVertexElementMutation(
          AuditAction.UPDATE, vertexMutation, vertex, "", user, lumifyVisibility.getVisibility());

      LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
          vertexMutation, visibilityJson, metadata, lumifyVisibility.getVisibility());

      this.graph.flush();

      workspaceRepository.updateEntityOnWorkspace(
          workspace, vertex.getId(), false, null, null, user);
    }

    // TODO: a better way to check if the same edge exists instead of looking it up every time?
    Edge edge =
        graph.addEdge(
            artifactVertex,
            vertex,
            this.artifactHasEntityIri,
            lumifyVisibility.getVisibility(),
            authorizations);
    LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
        edge, visibilityJson, metadata, lumifyVisibility.getVisibility(), authorizations);

    auditRepository.auditRelationship(
        AuditAction.CREATE,
        artifactVertex,
        vertex,
        edge,
        "",
        "",
        user,
        lumifyVisibility.getVisibility());

    TermMentionRowKey termMentionRowKey;
    if (termMentionRowKeyString != null) {
      termMentionRowKey = new TermMentionRowKey(RowKeyHelper.jsonDecode(termMentionRowKeyString));
    } else {
      termMentionRowKey =
          new TermMentionRowKey(
              artifactId, propertyKey, mentionStart, mentionEnd, edge.getId().toString());
    }
    TermMentionModel termMention = new TermMentionModel(termMentionRowKey);
    termMention
        .getMetadata()
        .setSign(title, lumifyVisibility.getVisibility())
        .setOntologyClassUri(concept.getDisplayName(), lumifyVisibility.getVisibility())
        .setConceptGraphVertexId(concept.getTitle(), lumifyVisibility.getVisibility())
        .setVertexId(vertex.getId().toString(), lumifyVisibility.getVisibility())
        .setEdgeId(edge.getId().toString(), lumifyVisibility.getVisibility());
    termMentionRepository.save(termMention);

    vertexMutation.addPropertyValue(
        graph.getIdGenerator().nextId().toString(),
        LumifyProperties.ROW_KEY.getPropertyName(),
        termMentionRowKey.toString(),
        metadata,
        lumifyVisibility.getVisibility());
    vertexMutation.save(authorizations);

    this.graph.flush();
    workQueueRepository.pushTextUpdated(artifactId);

    workQueueRepository.pushElement(edge);

    JSONObject result = new JSONObject();
    result.put("success", true);
    respondWithJson(response, result);
  }
}
public class RelationshipSetVisibility extends BaseRequestHandler {
  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(RelationshipSetVisibility.class);
  private final Graph graph;
  private final VisibilityTranslator visibilityTranslator;
  private final WorkQueueRepository workQueueRepository;
  private final AuditRepository auditRepository;

  @Inject
  public RelationshipSetVisibility(
      final Graph graph,
      final UserRepository userRepository,
      final Configuration configuration,
      final VisibilityTranslator visibilityTranslator,
      final WorkspaceRepository workspaceRepository,
      final WorkQueueRepository workQueueRepository,
      final AuditRepository auditRepository) {
    super(userRepository, workspaceRepository, configuration);
    this.graph = graph;
    this.visibilityTranslator = visibilityTranslator;
    this.workQueueRepository = workQueueRepository;
    this.auditRepository = auditRepository;
  }

  @Override
  public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
      throws Exception {
    final String graphEdgeId = getAttributeString(request, "graphEdgeId");
    final String visibilitySource = getRequiredParameter(request, "visibilitySource");

    User user = getUser(request);
    Authorizations authorizations = getAuthorizations(request, user);
    String workspaceId = getActiveWorkspaceId(request);

    Edge graphEdge = graph.getEdge(graphEdgeId, authorizations);
    if (graphEdge == null) {
      respondWithNotFound(response);
      return;
    }

    if (!graph.isVisibilityValid(new Visibility(visibilitySource), authorizations)) {
      LOGGER.warn(
          "%s is not a valid visibility for %s user", visibilitySource, user.getDisplayName());
      respondWithBadRequest(response, "visibilitySource", getString(request, "visibility.invalid"));
      chain.next(request, response);
      return;
    }

    LOGGER.info(
        "changing edge (%s) visibility source to %s",
        graphEdge.getId().toString(), visibilitySource);

    GraphUtil.VisibilityAndElementMutation<Edge> setPropertyResult =
        GraphUtil.updateElementVisibilitySource(
            visibilityTranslator,
            graphEdge,
            GraphUtil.getSandboxStatus(graphEdge, workspaceId),
            visibilitySource,
            workspaceId,
            authorizations);
    auditRepository.auditEdgeElementMutation(
        AuditAction.UPDATE,
        setPropertyResult.elementMutation,
        graphEdge,
        graphEdge.getVertex(Direction.OUT, authorizations),
        graphEdge.getVertex(Direction.IN, authorizations),
        "",
        user,
        setPropertyResult.visibility.getVisibility());

    this.graph.flush();

    this.workQueueRepository.pushGraphPropertyQueue(
        graphEdge,
        null,
        LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.getPropertyName(),
        workspaceId,
        visibilitySource);

    JSONObject json = JsonSerializer.toJson(graphEdge, workspaceId, authorizations);
    respondWithJson(response, json);
  }
}
Exemple #7
0
public class ImportMR extends LumifyMRBase {
  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ImportMR.class);
  public static final String WIKIPEDIA_MIME_TYPE = "text/plain";
  public static final String WIKIPEDIA_SOURCE = "Wikipedia";
  public static final String WIKIPEDIA_ID_PREFIX = "WIKIPEDIA_";
  public static final String WIKIPEDIA_LINK_ID_PREFIX = "WIKIPEDIA_LINK_";
  public static final String MULTI_VALUE_KEY = ImportMR.class.getName();

  private OntologyRepository ontologyRepository;

  static String getWikipediaPageVertexId(String pageTitle) {
    return WIKIPEDIA_ID_PREFIX + pageTitle.trim().toLowerCase();
  }

  static String getWikipediaPageToPageEdgeId(Vertex pageVertex, Vertex linkedPageVertex) {
    return WIKIPEDIA_LINK_ID_PREFIX
        + getWikipediaPageTitleFromId(pageVertex.getId())
        + "_"
        + getWikipediaPageTitleFromId(linkedPageVertex.getId());
  }

  static String getWikipediaPageTitleFromId(Object id) {
    return id.toString().substring(WIKIPEDIA_ID_PREFIX.length());
  }

  @Override
  protected void setupJob(Job job) throws Exception {
    verifyWikipediaPageConcept(ontologyRepository);
    verifyWikipediaPageInternalLinkWikipediaPageRelationship(ontologyRepository);

    job.setJarByClass(ImportMR.class);
    job.setMapperClass(ImportMRMapper.class);
    job.setNumReduceTasks(0);
    job.setMapOutputValueClass(Mutation.class);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(AccumuloElementOutputFormat.class);
    FileInputFormat.addInputPath(job, new Path(getConf().get("in")));
  }

  @Override
  protected String getJobName() {
    return "wikipediaImport";
  }

  private void verifyWikipediaPageInternalLinkWikipediaPageRelationship(
      OntologyRepository ontologyRepository) {
    Relationship wikipediaPageInternalLinkWikipediaPageRelationship =
        ontologyRepository.getRelationshipByIRI(
            WikipediaConstants.WIKIPEDIA_PAGE_INTERNAL_LINK_WIKIPEDIA_PAGE_CONCEPT_URI);
    if (wikipediaPageInternalLinkWikipediaPageRelationship == null) {
      throw new RuntimeException(
          WikipediaConstants.WIKIPEDIA_PAGE_INTERNAL_LINK_WIKIPEDIA_PAGE_CONCEPT_URI
              + " relationship not found");
    }
  }

  private void verifyWikipediaPageConcept(OntologyRepository ontologyRepository) {
    Concept wikipediaPageConcept =
        ontologyRepository.getConceptByIRI(WikipediaConstants.WIKIPEDIA_PAGE_CONCEPT_URI);
    if (wikipediaPageConcept == null) {
      throw new RuntimeException(
          WikipediaConstants.WIKIPEDIA_PAGE_CONCEPT_URI + " concept not found");
    }
  }

  @Override
  protected void parseArgs(JobConf conf, String[] args) {
    if (args.length != 1) {
      throw new RuntimeException("Required arguments <inputFileName>");
    }
    String inFileName = args[0];
    LOGGER.info("inFileName: %s", inFileName);
    conf.set("in", inFileName);
    conf.set(ImportMRMapper.CONFIG_SOURCE_FILE_NAME, new File(inFileName).getName());
  }

  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new ImportMR(), args);
    System.exit(res);
  }

  @Inject
  public void setOntologyRepository(OntologyRepository ontologyRepository) {
    this.ontologyRepository = ontologyRepository;
  }
}
Exemple #8
0
public class ArtifactRaw extends BaseRequestHandler {
  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ArtifactRaw.class);
  private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=([0-9]*)-([0-9]*)");

  private final Graph graph;

  @Inject
  public ArtifactRaw(
      final Graph graph,
      final UserRepository userRepository,
      final WorkspaceRepository workspaceRepository,
      final Configuration configuration) {
    super(userRepository, workspaceRepository, configuration);
    this.graph = graph;
  }

  @Override
  public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
      throws Exception {
    boolean download = getOptionalParameter(request, "download") != null;
    boolean playback = getOptionalParameter(request, "playback") != null;

    User user = getUser(request);
    Authorizations authorizations = getAuthorizations(request, user);

    String graphVertexId = UrlUtils.urlDecode(getAttributeString(request, "graphVertexId"));

    Vertex artifactVertex = graph.getVertex(graphVertexId, authorizations);
    if (artifactVertex == null) {
      respondWithNotFound(response);
      return;
    }

    String fileName = LumifyProperties.FILE_NAME.getPropertyValue(artifactVertex);
    if (fileName == null) {
      fileName = LumifyProperties.TITLE.getPropertyValue(artifactVertex);
    }

    if (playback) {
      handlePartialPlayback(request, response, artifactVertex, fileName, user);
    } else {
      String mimeType = getMimeType(artifactVertex);
      response.setContentType(mimeType);
      setMaxAge(response, EXPIRES_1_HOUR);
      if (download) {
        response.addHeader("Content-Disposition", "attachment; filename=" + fileName);
      } else {
        response.addHeader("Content-Disposition", "inline; filename=" + fileName);
      }

      StreamingPropertyValue rawValue = LumifyProperties.RAW.getPropertyValue(artifactVertex);
      if (rawValue == null) {
        LOGGER.warn("Could not find raw on artifact: %s", artifactVertex.getId().toString());
        respondWithNotFound(response);
        return;
      }
      InputStream in = rawValue.getInputStream();
      try {
        IOUtils.copy(in, response.getOutputStream());
      } finally {
        in.close();
      }
    }

    chain.next(request, response);
  }

  private void handlePartialPlayback(
      HttpServletRequest request,
      HttpServletResponse response,
      Vertex artifactVertex,
      String fileName,
      User user)
      throws IOException {
    String type = getRequiredParameter(request, "type");

    InputStream in;
    Long totalLength = null;
    long partialStart = 0;
    Long partialEnd = null;
    String range = request.getHeader("Range");

    if (range != null) {
      response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);

      Matcher m = RANGE_PATTERN.matcher(range);
      if (m.matches()) {
        partialStart = Long.parseLong(m.group(1));
        if (m.group(2).length() > 0) {
          partialEnd = Long.parseLong(m.group(2));
        }
      }
    }

    response.setCharacterEncoding(null);
    response.setContentType(type);
    response.addHeader("Content-Disposition", "attachment; filename=" + fileName);

    StreamingPropertyValue mediaPropertyValue = getStreamingPropertyValue(artifactVertex, type);

    totalLength = mediaPropertyValue.getLength();
    in = mediaPropertyValue.getInputStream();

    if (partialEnd == null) {
      partialEnd = totalLength;
    }

    // Ensure that the last byte position is less than the instance-length
    partialEnd = Math.min(partialEnd, totalLength - 1);
    long partialLength = totalLength;

    if (range != null) {
      partialLength = partialEnd - partialStart + 1;
      response.addHeader(
          "Content-Range", "bytes " + partialStart + "-" + partialEnd + "/" + totalLength);
      if (partialStart > 0) {
        in.skip(partialStart);
      }
    }

    response.addHeader("Content-Length", "" + partialLength);

    OutputStream out = response.getOutputStream();
    copy(in, out, partialLength);

    response.flushBuffer();
  }

  private StreamingPropertyValue getStreamingPropertyValue(Vertex artifactVertex, String type) {
    StreamingPropertyValue mediaPropertyValue;
    if (MediaLumifyProperties.MIME_TYPE_AUDIO_MP4.equals(type)) {
      mediaPropertyValue = MediaLumifyProperties.AUDIO_MP4.getPropertyValue(artifactVertex);
      checkNotNull(
          mediaPropertyValue,
          String.format(
              "Could not find %s property on artifact %s",
              MediaLumifyProperties.AUDIO_MP4, artifactVertex.getId()));
    } else if (MediaLumifyProperties.MIME_TYPE_AUDIO_OGG.equals(type)) {
      mediaPropertyValue = MediaLumifyProperties.AUDIO_OGG.getPropertyValue(artifactVertex);
      checkNotNull(
          mediaPropertyValue,
          String.format(
              "Could not find %s property on artifact %s",
              MediaLumifyProperties.AUDIO_OGG, artifactVertex.getId()));
    } else if (MediaLumifyProperties.MIME_TYPE_VIDEO_MP4.equals(type)) {
      mediaPropertyValue = MediaLumifyProperties.VIDEO_MP4.getPropertyValue(artifactVertex);
      checkNotNull(
          mediaPropertyValue,
          String.format(
              "Could not find %s property on artifact %s",
              MediaLumifyProperties.VIDEO_MP4, artifactVertex.getId()));
    } else if (MediaLumifyProperties.MIME_TYPE_VIDEO_WEBM.equals(type)) {
      mediaPropertyValue = MediaLumifyProperties.VIDEO_WEBM.getPropertyValue(artifactVertex);
      checkNotNull(
          mediaPropertyValue,
          String.format(
              "Could not find %s property on artifact %s",
              MediaLumifyProperties.VIDEO_WEBM, artifactVertex.getId()));
    } else {
      throw new RuntimeException("Invalid video type: " + type);
    }
    return mediaPropertyValue;
  }

  private void copy(InputStream in, OutputStream out, Long length) throws IOException {
    byte[] buffer = new byte[1024];
    int read = 0;
    while (length > 0 && (read = in.read(buffer, 0, (int) Math.min(length, buffer.length))) > 0) {
      out.write(buffer, 0, read);
      length -= read;
    }
  }

  private String getMimeType(Vertex artifactVertex) {
    String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(artifactVertex);
    if (mimeType == null || mimeType.isEmpty()) {
      mimeType = "application/octet-stream";
    }
    return mimeType;
  }
}
public class GraphVertexSearch extends BaseRequestHandler {
  private final int DEFAULT_RESULT_COUNT = 100;

  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(GraphVertexSearch.class);
  private final Graph graph;
  private final OntologyRepository ontologyRepository;
  private final DetectedObjectRepository detectedObjectRepository;
  private final UserRepository userRepository;

  @Inject
  public GraphVertexSearch(
      final OntologyRepository ontologyRepository,
      final Graph graph,
      final UserRepository userRepository,
      final Configuration configuration,
      final WorkspaceRepository workspaceRepository,
      final DetectedObjectRepository detectedObjectRepository) {
    super(userRepository, workspaceRepository, configuration);
    this.ontologyRepository = ontologyRepository;
    this.graph = graph;
    this.detectedObjectRepository = detectedObjectRepository;
    this.userRepository = userRepository;
  }

  @Override
  public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
      throws Exception {
    long totalStartTime = System.nanoTime();
    final String query;
    final String filter = getRequiredParameter(request, "filter");
    final int offset = (int) getOptionalParameterLong(request, "offset", 0);
    final int size = (int) getOptionalParameterLong(request, "size", DEFAULT_RESULT_COUNT);
    final String conceptType = getOptionalParameter(request, "conceptType");
    final String getLeafNodes = getOptionalParameter(request, "leafNodes");
    final String relatedToVertexId = getOptionalParameter(request, "relatedToVertexId");
    if (relatedToVertexId == null) {
      query = getRequiredParameter(request, "q");
    } else {
      query = getOptionalParameter(request, "q");
    }

    long startTime = System.nanoTime();

    User user = getUser(request);
    Authorizations authorizations = getAuthorizations(request, user);
    String workspaceId = getActiveWorkspaceId(request);
    ModelUserContext modelUserContext =
        userRepository.getModelUserContext(authorizations, workspaceId);

    JSONArray filterJson = new JSONArray(filter);

    ontologyRepository.resolvePropertyIds(filterJson);

    graph.flush();

    LOGGER.debug("search %s\n%s", query, filterJson.toString(2));

    Query graphQuery;
    if (relatedToVertexId == null) {
      graphQuery = graph.query(query, authorizations);
    } else if (query == null || StringUtils.isBlank(query)) {
      graphQuery = graph.getVertex(relatedToVertexId, authorizations).query(authorizations);
    } else {
      graphQuery = graph.getVertex(relatedToVertexId, authorizations).query(query, authorizations);
    }

    for (int i = 0; i < filterJson.length(); i++) {
      JSONObject obj = filterJson.getJSONObject(i);
      if (obj.length() > 0) {
        updateQueryWithFilter(graphQuery, obj);
      }
    }

    if (conceptType != null) {
      Concept concept = ontologyRepository.getConceptByIRI(conceptType);
      if (getLeafNodes == null || !getLeafNodes.equals("false")) {
        List<Concept> leafNodeList = ontologyRepository.getAllLeafNodesByConcept(concept);
        if (leafNodeList.size() > 0) {
          String[] conceptIds = new String[leafNodeList.size()];
          int count = 0;
          for (Concept c : leafNodeList) {
            conceptIds[count] = c.getTitle();
            count++;
          }
          graphQuery.has(CONCEPT_TYPE.getPropertyName(), Compare.IN, conceptIds);
        }
      } else {
        graphQuery.has(CONCEPT_TYPE.getPropertyName(), conceptType);
      }
    }

    graphQuery.limit(size);
    graphQuery.skip(offset);
    Iterable<Vertex> searchResults;
    try {
      searchResults = graphQuery.vertices();
    } catch (SearchPhaseExecutionException ex) {
      respondWithBadRequest(response, "q", "Invalid Query");
      return;
    }

    Map<Object, Double> scores = null;
    if (searchResults instanceof IterableWithScores) {
      scores = ((IterableWithScores) searchResults).getScores();
    }

    long retrievalStartTime = System.nanoTime();
    List<JSONObject> verticesJsonList = new ArrayList<JSONObject>();
    for (Vertex vertex : searchResults) {
      JSONObject vertexJson = JsonSerializer.toJson(vertex, workspaceId);
      if (scores != null) {
        vertexJson.put("score", scores.get(vertex.getId()));
      }
      vertexJson.put(
          "detectedObjects",
          detectedObjectRepository.toJSON(vertex, modelUserContext, authorizations, workspaceId));
      verticesJsonList.add(vertexJson);
    }
    long retrievalEndTime = System.nanoTime();

    Collections.sort(
        verticesJsonList,
        new Comparator<JSONObject>() {
          @Override
          public int compare(JSONObject o1, JSONObject o2) {
            double score1 = o1.optDouble("score", 0.0);
            double score2 = o2.optDouble("score", 0.0);
            return -Double.compare(score1, score2);
          }
        });

    JSONArray verticesJson = new JSONArray();
    for (JSONObject vertexJson : verticesJsonList) {
      verticesJson.put(vertexJson);
    }

    long totalEndTime = System.nanoTime();

    JSONObject results = new JSONObject();
    results.put("vertices", verticesJson);
    results.put("nextOffset", offset + size);
    results.put("retrievalTime", retrievalEndTime - retrievalStartTime);
    results.put("totalTime", totalEndTime - totalStartTime);

    if (searchResults instanceof IterableWithTotalHits) {
      results.put("totalHits", ((IterableWithTotalHits) searchResults).getTotalHits());
    }
    if (searchResults instanceof IterableWithSearchTime) {
      results.put(
          "searchTime", ((IterableWithSearchTime) searchResults).getSearchTimeNanoSeconds());
    }

    long endTime = System.nanoTime();
    LOGGER.info(
        "Search for \"%s\" found %d vertices in %dms",
        query, verticesJsonList.size(), (endTime - startTime) / 1000 / 1000);

    respondWithJson(response, results);
  }

  private void updateQueryWithFilter(Query graphQuery, JSONObject obj) throws ParseException {
    String predicateString = obj.optString("predicate");
    JSONArray values = obj.getJSONArray("values");
    PropertyType propertyDataType = PropertyType.convert(obj.optString("propertyDataType"));
    String propertyName = obj.getString("propertyName");
    Object value0 = jsonValueToObject(values, propertyDataType, 0);

    if (PropertyType.STRING.equals(propertyDataType)
        && (predicateString == null || "".equals(predicateString))) {
      graphQuery.has(propertyName, TextPredicate.CONTAINS, value0);
    } else if (PropertyType.BOOLEAN.equals(propertyDataType)
        && (predicateString == null || "".equals(predicateString))) {
      graphQuery.has(propertyName, Compare.EQUAL, value0);
    } else if ("<".equals(predicateString)) {
      graphQuery.has(propertyName, Compare.LESS_THAN, value0);
    } else if (">".equals(predicateString)) {
      graphQuery.has(propertyName, Compare.GREATER_THAN, value0);
    } else if ("range".equals(predicateString)) {
      graphQuery.has(propertyName, Compare.GREATER_THAN_EQUAL, value0);
      graphQuery.has(
          propertyName, Compare.LESS_THAN_EQUAL, jsonValueToObject(values, propertyDataType, 1));
    } else if ("=".equals(predicateString) || "equal".equals(predicateString)) {
      graphQuery.has(propertyName, Compare.EQUAL, value0);
    } else if (PropertyType.GEO_LOCATION.equals(propertyDataType)) {
      graphQuery.has(propertyName, GeoCompare.WITHIN, value0);
    } else {
      throw new LumifyException("unhandled query\n" + obj.toString(2));
    }
  }

  private Object jsonValueToObject(JSONArray values, PropertyType propertyDataType, int index)
      throws ParseException {
    return OntologyProperty.convert(values, propertyDataType, index);
  }
}
public abstract class RegexGraphPropertyWorker extends GraphPropertyWorker {
  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(RegexGraphPropertyWorker.class);
  private final Pattern pattern;

  public RegexGraphPropertyWorker(String regEx) {
    this.pattern = Pattern.compile(regEx, Pattern.MULTILINE);
  }

  protected abstract String getOntologyClassUri();

  @Override
  public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
    super.prepare(workerPrepareData);
    LOGGER.debug(
        "Extractor prepared for entity type [%s] with regular expression: %s",
        getOntologyClassUri(), this.pattern.toString());
  }

  @Override
  public void execute(InputStream in, GraphPropertyWorkData data) throws Exception {
    LOGGER.debug("Extracting pattern [%s] from provided text", pattern);

    final String text = CharStreams.toString(new InputStreamReader(in, Charsets.UTF_8));

    final Matcher matcher = pattern.matcher(text);

    List<TermMention> termMentions = new ArrayList<TermMention>();
    while (matcher.find()) {
      TermMention termMention =
          createTerm(matcher, data.getProperty().getKey(), data.getVisibility());
      termMentions.add(termMention);
    }
    Vertex v = (Vertex) data.getElement();
    getAuditRepository()
        .auditAnalyzedBy(
            AuditAction.ANALYZED_BY, v, getClass().getSimpleName(), getUser(), v.getVisibility());
    saveTermMentions((Vertex) data.getElement(), termMentions);
  }

  private TermMention createTerm(final Matcher matched, String propertyKey, Visibility visibility) {
    final String patternGroup = matched.group();
    int start = matched.start();
    int end = matched.end();

    return new TermMention.Builder(
            start, end, patternGroup, getOntologyClassUri(), propertyKey, visibility)
        .resolved(false)
        .useExisting(true)
        .process(getClass().getName())
        .build();
  }

  @Override
  public boolean isHandled(Element element, Property property) {
    if (property == null) {
      return false;
    }

    if (property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
      return false;
    }

    String mimeType =
        (String) property.getMetadata().get(LumifyProperties.MIME_TYPE.getPropertyName());
    return !(mimeType == null || !mimeType.startsWith("text"));
  }
}
@Parameters(
    separators = "=",
    commandDescription = "Exports Palantir Oracle database to hadoop SequenceFiles")
public class DataToSequenceFile implements Exporter.ExporterSource {
  public static final String ONTOLOGY_XML_DIR_NAME = "OntologyXML";

  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(DataToSequenceFile.class);

  @Parameter(
      names = {"-n", "--namespace"},
      description = "Table namespace",
      required = true)
  private String tableNamespace;

  @Parameter(
      names = {"-c", "--connectionstring"},
      description = "Database connection string",
      required = true)
  private String databaseConnectionString;

  @Parameter(
      names = {"-u", "--username"},
      description = "Database username",
      required = true)
  private String databaseUsername;

  @Parameter(
      names = {"-p", "--password"},
      description = "Database password",
      required = true)
  private String databasePassword;

  @Parameter(
      names = {"-d", "--dest"},
      description = "Destination path hadoop url",
      required = true)
  private String destination;

  @Parameter(
      names = {"--baseiri"},
      description = "base IRI for ontology",
      required = true)
  private String baseIri;

  @Parameter(
      names = {"--exporters"},
      description = "comma separated list of exporters (default: all)")
  private String exporters = "all";

  private Path destinationPath;
  private FileSystem fs;
  private SqlRunner sqlRunner;
  private Configuration hadoopConfiguration;
  public static final List<Exporter> EXPORTERS = new ArrayList<>();

  static {
    EXPORTERS.add(new PtObjectTypeExporter());
    EXPORTERS.add(new PtPropertyTypeExporter());
    EXPORTERS.add(new PtLinkTypeExporter());
    EXPORTERS.add(new PtTypeGroupExporter());
    EXPORTERS.add(new PtNodeDisplayTypeExporter());
    EXPORTERS.add(new PtImageInfoExporter());
    EXPORTERS.add(new PtOntologyResourceExporter());
    EXPORTERS.add(new PtLinkRelationExporter());
    EXPORTERS.add(new PtUserExporter());
    EXPORTERS.add(new PtGraphExporter());
    EXPORTERS.add(new PtObjectExporter());
    EXPORTERS.add(new PtGraphObjectExporter());
    EXPORTERS.add(new PtObjectObjectExporter());
    EXPORTERS.add(new PtMediaAndValueExporter());
    EXPORTERS.add(new PtPropertyAndValueExporter());
    EXPORTERS.add(new PtNoteAndNoteValueExporter());
  }

  public DataToSequenceFile(String[] args) {
    new JCommander(this, args);
    EXPORTERS.add(new OntologyToOwl(baseIri));

    if (exporters.equalsIgnoreCase("all")) {
      exporters =
          Joiner.on(',')
              .join(
                  new ConvertingIterable<Exporter, String>(EXPORTERS) {
                    @Override
                    protected String convert(Exporter o) {
                      return o.getObjectClass().getSimpleName();
                    }
                  });
    }
  }

  public static void main(String[] args) throws Exception {
    try {
      new DataToSequenceFile(args).run();
    } catch (Throwable ex) {
      LOGGER.error("Failed to export", ex);
    }
  }

  private void run() throws Exception {
    LOGGER.info("begin export");
    List<Exporter> exportersToRun = getExportersToRun(exporters);
    for (Exporter exporter : exportersToRun) {
      LOGGER.info("Preparing to run: %s", exporter.getClass().getSimpleName());
    }

    hadoopConfiguration = new Configuration(true);
    destinationPath = new Path(destination);
    fs = getFileSystem(hadoopConfiguration);
    sqlRunner = createSqlRunner();
    sqlRunner.connect();
    try {
      for (Exporter exporter : exportersToRun) {
        LOGGER.info("Running: %s", exporter.getClass().getSimpleName());
        exporter.run(this);
      }
    } finally {
      sqlRunner.close();
    }
    LOGGER.info("export complete!");
  }

  private List<Exporter> getExportersToRun(String exporters) {
    List<Exporter> results = new ArrayList<>();
    for (String exporterString : exporters.split(",")) {
      Exporter e = null;
      for (Exporter exporter : EXPORTERS) {
        if (exporter.getObjectClass().getSimpleName().equalsIgnoreCase(exporterString)) {
          e = exporter;
          break;
        }
      }
      if (e == null) {
        throw new RuntimeException("invalid exporter: " + exporterString);
      }
      results.add(e);
    }
    return results;
  }

  private SqlRunner createSqlRunner() {
    return new SqlRunner(
        databaseConnectionString, databaseUsername, databasePassword, tableNamespace);
  }

  private FileSystem getFileSystem(Configuration hadoopConfiguration)
      throws IOException, URISyntaxException {
    FileSystem fs = FileSystem.get(new URI(destination), hadoopConfiguration);
    fs.mkdirs(new Path(destination));
    return fs;
  }

  public Path getDestinationPath() {
    return destinationPath;
  }

  public SqlRunner getSqlRunner() {
    return sqlRunner;
  }

  @Override
  public FileSystem getFileSystem() {
    return fs;
  }

  public Configuration getHadoopConfiguration() {
    return hadoopConfiguration;
  }

  public void writeFile(String fileName, String data) {
    Path path = new Path(getDestinationPath(), fileName);
    try {
      try (FSDataOutputStream out = getFileSystem().create(path, true)) {
        out.write(data.getBytes());
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not write file: " + path, e);
    }
  }
}
public abstract class GraphPropertyWorker {
  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(GraphPropertyWorker.class);
  private Graph graph;
  private WorkQueueRepository workQueueRepository;
  private WorkspaceRepository workspaceRepository;
  private OntologyRepository ontologyRepository;
  private AuditRepository auditRepository;
  private TermMentionRepository termMentionRepository;
  private GraphPropertyWorkerPrepareData workerPrepareData;
  private Configuration configuration;
  private String artifactHasEntityIri;
  private String locationIri;
  private String organizationIri;
  private String personIri;
  private VisibilityTranslator visibilityTranslator;

  public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
    this.workerPrepareData = workerPrepareData;
    this.artifactHasEntityIri =
        getConfiguration().get(Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
    if (this.artifactHasEntityIri == null) {
      throw new LumifyException(
          "Could not find configuration for " + Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
    }

    this.locationIri =
        (String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_LOCATION);
    if (this.locationIri == null || this.locationIri.length() == 0) {
      throw new LumifyException(
          "Could not find configuration: " + Configuration.ONTOLOGY_IRI_LOCATION);
    }

    this.organizationIri =
        (String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_ORGANIZATION);
    if (this.organizationIri == null || this.organizationIri.length() == 0) {
      throw new LumifyException(
          "Could not find configuration: " + Configuration.ONTOLOGY_IRI_ORGANIZATION);
    }

    this.personIri =
        (String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_PERSON);
    if (this.personIri == null || this.personIri.length() == 0) {
      throw new LumifyException(
          "Could not find configuration: " + Configuration.ONTOLOGY_IRI_PERSON);
    }
  }

  public abstract void execute(InputStream in, GraphPropertyWorkData data) throws Exception;

  public abstract boolean isHandled(Element element, Property property);

  public boolean isLocalFileRequired() {
    return false;
  }

  protected User getUser() {
    return this.workerPrepareData.getUser();
  }

  public Authorizations getAuthorizations() {
    return this.workerPrepareData.getAuthorizations();
  }

  @Inject
  public final void setGraph(Graph graph) {
    this.graph = graph;
  }

  protected Graph getGraph() {
    return graph;
  }

  @Inject
  public final void setWorkQueueRepository(WorkQueueRepository workQueueRepository) {
    this.workQueueRepository = workQueueRepository;
  }

  @Inject
  public final void setWorkspaceRepository(WorkspaceRepository workspaceRepository) {
    this.workspaceRepository = workspaceRepository;
  }

  protected WorkQueueRepository getWorkQueueRepository() {
    return workQueueRepository;
  }

  protected OntologyRepository getOntologyRepository() {
    return ontologyRepository;
  }

  @Inject
  public final void setOntologyRepository(OntologyRepository ontologyRepository) {
    this.ontologyRepository = ontologyRepository;
  }

  protected AuditRepository getAuditRepository() {
    return auditRepository;
  }

  @Inject
  public final void setAuditRepository(AuditRepository auditRepository) {
    this.auditRepository = auditRepository;
  }

  @Inject
  public final void setVisibilityTranslatory(VisibilityTranslator visibilityTranslator) {
    this.visibilityTranslator = visibilityTranslator;
  }

  protected TermMentionRepository getTermMentionRepository() {
    return termMentionRepository;
  }

  @Inject
  public void setTermMentionRepository(TermMentionRepository termMentionRepository) {
    this.termMentionRepository = termMentionRepository;
  }

  protected Configuration getConfiguration() {
    return configuration;
  }

  @Inject
  public final void setConfiguration(Configuration configuration) {
    this.configuration = configuration;
  }

  /** Determines if this is a property that should be analyzed by text processing tools. */
  protected boolean isTextProperty(Property property) {
    if (property == null) {
      return false;
    }

    if (property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
      return false;
    }

    String mimeType =
        (String) property.getMetadata().get(LumifyProperties.MIME_TYPE.getPropertyName());
    return !(mimeType == null || !mimeType.startsWith("text"));
  }

  protected String mapToOntologyIri(String type) {
    String ontologyClassUri;
    if ("location".equals(type)) {
      ontologyClassUri = this.locationIri;
    } else if ("organization".equals(type)) {
      ontologyClassUri = this.organizationIri;
    } else if ("person".equals(type)) {
      ontologyClassUri = this.personIri;
    } else {
      ontologyClassUri = LumifyProperties.CONCEPT_TYPE_THING;
    }
    return ontologyClassUri;
  }

  protected void addVideoTranscriptAsTextPropertiesToMutation(
      ExistingElementMutation<Vertex> mutation,
      String propertyKey,
      VideoTranscript videoTranscript,
      Map<String, Object> metadata,
      Visibility visibility) {
    metadata.put(LumifyProperties.META_DATA_MIME_TYPE, "text/plain");
    for (VideoTranscript.TimedText entry : videoTranscript.getEntries()) {
      String textPropertyKey = getVideoTranscriptTimedTextPropertyKey(propertyKey, entry);
      StreamingPropertyValue value =
          new StreamingPropertyValue(
              new ByteArrayInputStream(entry.getText().getBytes()), String.class);
      LumifyProperties.TEXT.addPropertyValue(
          mutation, textPropertyKey, value, metadata, visibility);
    }
  }

  protected void pushVideoTranscriptTextPropertiesOnWorkQueue(
      Element element, String propertyKey, VideoTranscript videoTranscript) {
    for (VideoTranscript.TimedText entry : videoTranscript.getEntries()) {
      String textPropertyKey = getVideoTranscriptTimedTextPropertyKey(propertyKey, entry);
      getWorkQueueRepository()
          .pushGraphPropertyQueue(
              element, textPropertyKey, LumifyProperties.TEXT.getPropertyName());
    }
  }

  private String getVideoTranscriptTimedTextPropertyKey(
      String propertyKey, VideoTranscript.TimedText entry) {
    String startTime = String.format("%08d", Math.max(0L, entry.getTime().getStart()));
    String endTime = String.format("%08d", Math.max(0L, entry.getTime().getEnd()));
    return propertyKey
        + RowKeyHelper.MINOR_FIELD_SEPARATOR
        + MediaLumifyProperties.VIDEO_FRAME.getPropertyName()
        + RowKeyHelper.MINOR_FIELD_SEPARATOR
        + startTime
        + RowKeyHelper.MINOR_FIELD_SEPARATOR
        + endTime;
  }

  protected void saveTermExtractionResult(
      Vertex artifactGraphVertex,
      TermExtractionResult termExtractionResult,
      String workspaceId,
      String visibilitySource) {
    List<TermMentionWithGraphVertex> termMentionsResults =
        saveTermMentions(
            artifactGraphVertex,
            termExtractionResult.getTermMentions(),
            workspaceId,
            visibilitySource);
    saveRelationships(termExtractionResult.getRelationships(), termMentionsResults);
    saveVertexRelationships(termExtractionResult.getVertexRelationships(), termMentionsResults);
  }

  private void saveVertexRelationships(
      List<VertexRelationship> vertexRelationships,
      List<TermMentionWithGraphVertex> termMentionsWithGraphVertices) {
    for (VertexRelationship vertexRelationship : vertexRelationships) {
      TermMentionWithGraphVertex sourceTermMentionsWithGraphVertex =
          findTermMentionWithGraphVertex(
              termMentionsWithGraphVertices, vertexRelationship.getSource());
      checkNotNull(sourceTermMentionsWithGraphVertex, "Could not find source");
      checkNotNull(sourceTermMentionsWithGraphVertex.getVertex(), "Could not find source vertex");

      Vertex targetVertex = graph.getVertex(vertexRelationship.getTargetId(), getAuthorizations());
      checkNotNull(
          targetVertex, "Could not find target vertex: " + vertexRelationship.getTargetId());

      String label = vertexRelationship.getLabel();
      checkNotNull(label, "label is required");

      Edge edge =
          singleOrDefault(
              sourceTermMentionsWithGraphVertex
                  .getVertex()
                  .getEdges(targetVertex, Direction.OUT, label, getAuthorizations()),
              null);
      if (edge == null) {
        LOGGER.debug(
            "adding edge %s -> %s (%s)",
            sourceTermMentionsWithGraphVertex.getVertex().getId(), targetVertex.getId(), label);
        graph.addEdge(
            sourceTermMentionsWithGraphVertex.getVertex(),
            targetVertex,
            label,
            vertexRelationship.getVisibility(),
            getAuthorizations());
      }
    }
  }

  private void saveRelationships(
      List<TermRelationship> relationships,
      List<TermMentionWithGraphVertex> termMentionsWithGraphVertices) {
    for (TermRelationship relationship : relationships) {
      TermMentionWithGraphVertex sourceTermMentionsWithGraphVertex =
          findTermMentionWithGraphVertex(
              termMentionsWithGraphVertices, relationship.getSourceTermMention());
      checkNotNull(
          sourceTermMentionsWithGraphVertex,
          "source was not found for " + relationship.getSourceTermMention());
      checkNotNull(
          sourceTermMentionsWithGraphVertex.getVertex(),
          "source vertex was not found for " + relationship.getSourceTermMention());
      TermMentionWithGraphVertex destTermMentionsWithGraphVertex =
          findTermMentionWithGraphVertex(
              termMentionsWithGraphVertices, relationship.getDestTermMention());
      checkNotNull(
          destTermMentionsWithGraphVertex,
          "dest was not found for " + relationship.getDestTermMention());
      checkNotNull(
          destTermMentionsWithGraphVertex.getVertex(),
          "dest vertex was not found for " + relationship.getDestTermMention());
      String label = relationship.getLabel();

      // TODO: a better way to check if the same edge exists instead of looking it up every time?
      Edge edge =
          singleOrDefault(
              sourceTermMentionsWithGraphVertex
                  .getVertex()
                  .getEdges(
                      destTermMentionsWithGraphVertex.getVertex(),
                      Direction.OUT,
                      label,
                      getAuthorizations()),
              null);
      if (edge == null) {
        graph.addEdge(
            sourceTermMentionsWithGraphVertex.getVertex(),
            destTermMentionsWithGraphVertex.getVertex(),
            label,
            relationship.getVisibility(),
            getAuthorizations());
      }
    }
    graph.flush();
  }

  private TermMentionWithGraphVertex findTermMentionWithGraphVertex(
      List<TermMentionWithGraphVertex> termMentionsWithGraphVertices, TermMention termMention) {
    for (TermMentionWithGraphVertex termMentionsWithGraphVertex : termMentionsWithGraphVertices) {
      if (termMentionsWithGraphVertex.getTermMention().getRowKey().getStartOffset()
              == termMention.getStart()
          && termMentionsWithGraphVertex.getTermMention().getRowKey().getEndOffset()
              == termMention.getEnd()
          && termMentionsWithGraphVertex.getVertex() != null) {
        return termMentionsWithGraphVertex;
      }
    }
    return null;
  }

  protected List<TermMentionWithGraphVertex> saveTermMentions(
      Vertex artifactGraphVertex,
      Iterable<TermMention> termMentions,
      String workspaceId,
      String visibilitySource) {
    getAuditRepository()
        .auditAnalyzedBy(
            AuditAction.ANALYZED_BY,
            artifactGraphVertex,
            getClass().getSimpleName(),
            getUser(),
            artifactGraphVertex.getVisibility());
    for (TermMentionFilter termMentionFilter : this.workerPrepareData.getTermMentionFilters()) {
      try {
        termMentions = termMentionFilter.apply(artifactGraphVertex, termMentions);
      } catch (Exception ex) {
        throw new LumifyException(
            "Failed to run term mention filter: " + termMentionFilter.getClass().getName(), ex);
      }
    }

    List<TermMentionWithGraphVertex> results = new ArrayList<TermMentionWithGraphVertex>();
    for (TermMention termMention : termMentions) {
      results.add(saveTermMention(artifactGraphVertex, termMention, workspaceId, visibilitySource));
    }
    return results;
  }

  private TermMentionWithGraphVertex saveTermMention(
      Vertex artifactGraphVertex,
      TermMention termMention,
      String workspaceId,
      String visibilitySource) {
    LOGGER.debug(
        "Saving term mention '%s':%s:%s (%d:%d)",
        termMention.getSign(),
        termMention.getOntologyClassUri(),
        termMention.getPropertyKey(),
        termMention.getStart(),
        termMention.getEnd());
    Vertex vertex = null;
    if (visibilitySource == null) {
      visibilitySource = termMention.getVisibility().getVisibilityString();
    }

    JSONObject visibilityJson = new JSONObject();
    visibilityJson.put(VisibilityTranslator.JSON_SOURCE, visibilitySource);
    Visibility visibility = visibilityTranslator.toVisibility(visibilityJson).getVisibility();

    Visibility visibilityWithWorkspaceId;
    JSONObject visibilityJsonWithWorkspaceId;
    if (!workspaceId.equals("")) {
      visibilityJsonWithWorkspaceId =
          GraphUtil.updateVisibilitySourceAndAddWorkspaceId(null, visibilitySource, workspaceId);
      visibilityWithWorkspaceId =
          visibilityTranslator.toVisibility(visibilityJsonWithWorkspaceId).getVisibility();
    } else {
      visibilityWithWorkspaceId = visibility;
      visibilityJsonWithWorkspaceId = visibilityJson;
    }

    TermMentionRowKey termMentionRowKey =
        new TermMentionRowKey(
            artifactGraphVertex.getId().toString(),
            termMention.getPropertyKey(),
            termMention.getStart(),
            termMention.getEnd());
    TermMentionModel termMentionModel = new TermMentionModel(termMentionRowKey);
    termMentionModel.getMetadata().setSign(termMention.getSign(), visibility);
    termMentionModel
        .getMetadata()
        .setOntologyClassUri(termMention.getOntologyClassUri(), visibility);
    if (termMention.getProcess() != null && !termMention.getProcess().equals("")) {
      termMentionModel.getMetadata().setAnalyticProcess(termMention.getProcess(), visibility);
    }

    Concept concept = ontologyRepository.getConceptByIRI(termMention.getOntologyClassUri());
    if (concept == null) {
      LOGGER.error("Could not find ontology graph vertex '%s'", termMention.getOntologyClassUri());
      return null;
    }
    termMentionModel.getMetadata().setConceptGraphVertexId(concept.getTitle(), visibility);

    if (termMention.isResolved()) {
      String title = termMention.getSign();
      ElementMutation<Vertex> vertexElementMutation;
      if (termMention.getUseExisting()) {
        graph.flush(); // make sure the previous term mentions have made it into the graph
        if (termMention.getId() != null) {
          vertex = graph.getVertex(termMention.getId(), getAuthorizations());
        } else {
          vertex =
              singleOrDefault(
                  graph
                      .query(getAuthorizations())
                      .has(LumifyProperties.TITLE.getPropertyName(), title)
                      .has(LumifyProperties.CONCEPT_TYPE.getPropertyName(), concept.getTitle())
                      .vertices(),
                  null);
        }
      }

      Map<String, Object> metadata = new HashMap<String, Object>();
      LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setMetadata(
          metadata, visibilityJsonWithWorkspaceId);

      if (vertex == null) {
        if (termMention.getId() != null) {
          vertexElementMutation =
              graph.prepareVertex(termMention.getId(), visibilityWithWorkspaceId);
        } else {
          vertexElementMutation = graph.prepareVertex(visibilityWithWorkspaceId);
        }
        LumifyProperties.TITLE.setProperty(
            vertexElementMutation, title, metadata, visibilityWithWorkspaceId);
        LumifyProperties.CONCEPT_TYPE.setProperty(
            vertexElementMutation, concept.getTitle(), metadata, visibilityWithWorkspaceId);
      } else {
        vertexElementMutation = vertex.prepareMutation();
      }

      for (TermMention.TermMentionProperty termMentionProperty : termMention.getNewProperties()) {
        vertexElementMutation.addPropertyValue(
            termMentionProperty.getKey(),
            termMentionProperty.getName(),
            termMentionProperty.getValue(),
            metadata,
            visibilityWithWorkspaceId);
      }

      LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
          vertexElementMutation,
          visibilityJsonWithWorkspaceId,
          metadata,
          visibilityWithWorkspaceId);
      vertexElementMutation.addPropertyValue(
          graph.getIdGenerator().nextId(),
          LumifyProperties.ROW_KEY.getPropertyName(),
          termMentionRowKey.toString(),
          metadata,
          visibilityWithWorkspaceId);

      if (!(vertexElementMutation instanceof ExistingElementMutation)) {
        vertex = vertexElementMutation.save(getAuthorizations());
        auditRepository.auditVertexElementMutation(
            AuditAction.UPDATE,
            vertexElementMutation,
            vertex,
            termMention.getProcess(),
            getUser(),
            visibilityWithWorkspaceId);
      } else {
        auditRepository.auditVertexElementMutation(
            AuditAction.UPDATE,
            vertexElementMutation,
            vertex,
            termMention.getProcess(),
            getUser(),
            visibilityWithWorkspaceId);
        vertex = vertexElementMutation.save(getAuthorizations());
      }

      // TODO: a better way to check if the same edge exists instead of looking it up every time?
      Edge edge =
          singleOrDefault(
              artifactGraphVertex.getEdges(
                  vertex, Direction.OUT, artifactHasEntityIri, getAuthorizations()),
              null);
      if (edge == null) {
        edge =
            graph.addEdge(
                artifactGraphVertex, vertex, artifactHasEntityIri, visibility, getAuthorizations());
        LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
            edge,
            visibilityJsonWithWorkspaceId,
            metadata,
            visibilityWithWorkspaceId,
            getAuthorizations());
        auditRepository.auditRelationship(
            AuditAction.CREATE,
            artifactGraphVertex,
            vertex,
            edge,
            termMention.getProcess(),
            "",
            getUser(),
            visibilityWithWorkspaceId);
      }

      graph.flush();
      if (workspaceId != null && !workspaceId.equals("")) {
        Workspace workspace = workspaceRepository.findById(workspaceId, getUser());
        workspaceRepository.updateEntityOnWorkspace(
            workspace, vertex.getId(), null, null, null, getUser());
      }

      termMentionModel
          .getMetadata()
          .setVertexId(vertex.getId().toString(), visibility)
          .setEdgeId(edge.getId().toString(), visibility);
    }

    getTermMentionRepository().save(termMentionModel, FlushFlag.NO_FLUSH);
    return new TermMentionWithGraphVertex(termMentionModel, vertex);
  }

  public static class TermMentionWithGraphVertex {
    private final TermMentionModel termMention;
    private final Vertex vertex;

    public TermMentionWithGraphVertex(TermMentionModel termMention, Vertex vertex) {
      this.termMention = termMention;
      this.vertex = vertex;
    }

    public TermMentionModel getTermMention() {
      return termMention;
    }

    public Vertex getVertex() {
      return vertex;
    }
  }
}
Exemple #13
0
public class ImportMRMapper extends ElementMapper<LongWritable, Text, Text, Mutation> {
  private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ImportMRMapper.class);
  public static final String CONFIG_SOURCE_FILE_NAME = "sourceFileName";
  private AccumuloGraph graph;
  private Visibility visibility;
  private Authorizations authorizations;
  private SystemUser user;
  private UserRepository userRepository;
  private SecureGraphAuditRepository auditRepository;
  private String sourceFileName;

  @Override
  protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Map configurationMap = SecureGraphMRUtils.toMap(context.getConfiguration());
    this.graph =
        (AccumuloGraph)
            new GraphFactory().createGraph(MapUtils.getAllWithPrefix(configurationMap, "graph"));
    this.visibility = new Visibility("");
    this.authorizations = new AccumuloAuthorizations();
    this.user = new SystemUser(null);
    VersionService versionService = new VersionService();
    io.lumify.core.config.Configuration configuration =
        new HashMapConfigurationLoader(configurationMap).createConfiguration();
    this.auditRepository =
        new SecureGraphAuditRepository(null, versionService, configuration, null, userRepository);
    this.sourceFileName = context.getConfiguration().get(CONFIG_SOURCE_FILE_NAME);
  }

  @Override
  protected void map(LongWritable filePosition, Text line, Context context)
      throws IOException, InterruptedException {
    try {
      safeMap(filePosition, line, context);
    } catch (Exception ex) {
      LOGGER.error("failed mapping " + filePosition, ex);
    }
  }

  private void safeMap(LongWritable filePosition, Text line, Context context)
      throws IOException, InterruptedException {
    String lineString = line.toString();
    int colonOffet = lineString.indexOf(':');
    if (colonOffet < 1) {
      return;
    }
    long userId = Long.parseLong(lineString.substring(0, colonOffet));
    context.setStatus("User: "******"Friendster MR", "", user, visibility);
    context.write(key, AccumuloSession.createMutationFromRow(audit));

    String friends = lineString.substring(colonOffet + 1).trim();
    if ("notfound".equals(friends) || "private".equals(friends)) {
      // do nothing?
    } else {
      String[] friendsArray = friends.split(",");
      for (String friend : friendsArray) {
        friend = friend.trim();
        if (friend.length() == 0) {
          continue;
        }
        long friendId = Long.parseLong(friend);
        Vertex friendVertex = createUserVertex(friendId);
        addEdge(
            ImportMR.getFriendEdgeId(userVertex, friendVertex),
            userVertex,
            friendVertex,
            FriendsterOntology.EDGE_LABEL_FRIEND,
            visibility,
            authorizations);
        context.getCounter(FriendsterImportCounters.FRIEND_EDGES_CREATED).increment(1);
      }
    }

    context.getCounter(FriendsterImportCounters.USERS_PROCESSED).increment(1);
  }

  private Vertex createUserVertex(long userId) {
    String userVertexId = ImportMR.getUserVertexId(userId);
    VertexBuilder userVertexBuilder = prepareVertex(userVertexId, visibility);
    LumifyProperties.CONCEPT_TYPE.setProperty(
        userVertexBuilder, FriendsterOntology.CONCEPT_TYPE_USER, visibility);
    Metadata titleMetadata = new Metadata();
    LumifyProperties.TITLE.addPropertyValue(
        userVertexBuilder,
        ImportMR.MULTI_VALUE_KEY,
        "Friendster User " + userId,
        titleMetadata,
        visibility);
    LumifyProperties.SOURCE.addPropertyValue(
        userVertexBuilder, ImportMR.MULTI_VALUE_KEY, ImportMR.FRIENDSTER_SOURCE, visibility);
    return userVertexBuilder.save(authorizations);
  }

  @Override
  protected void saveDataMutation(Context context, Text dataTableName, Mutation m)
      throws IOException, InterruptedException {
    context.write(ImportMR.getKey(dataTableName.toString(), m.getRow()), m);
  }

  @Override
  protected void saveEdgeMutation(Context context, Text edgesTableName, Mutation m)
      throws IOException, InterruptedException {
    context.write(ImportMR.getKey(edgesTableName.toString(), m.getRow()), m);
  }

  @Override
  protected void saveVertexMutation(Context context, Text verticesTableName, Mutation m)
      throws IOException, InterruptedException {
    context.write(ImportMR.getKey(verticesTableName.toString(), m.getRow()), m);
  }

  @Override
  public IdGenerator getIdGenerator() {
    return this.graph.getIdGenerator();
  }

  @Inject
  public void setUserRepository(UserRepository userRepository) {
    this.userRepository = userRepository;
  }
}
public abstract class OntologyRepositoryBase implements OntologyRepository {
  private static final LumifyLogger LOGGER =
      LumifyLoggerFactory.getLogger(OntologyRepositoryBase.class);

  public void defineOntology(Authorizations authorizations) {
    Concept rootConcept = getOrCreateConcept(null, OntologyRepository.ROOT_CONCEPT_IRI, "root");
    Concept entityConcept =
        getOrCreateConcept(rootConcept, OntologyRepository.ENTITY_CONCEPT_IRI, "thing");
    addEntityGlyphIcon(entityConcept);
    importBaseOwlFile(authorizations);
  }

  private void importBaseOwlFile(Authorizations authorizations) {
    importResourceOwl("base.owl", "http://lumify.io", authorizations);
    importResourceOwl("user.owl", "http://lumify.io/user", authorizations);
    importResourceOwl("workspace.owl", "http://lumify.io/workspace", authorizations);
  }

  private void importResourceOwl(String fileName, String iri, Authorizations authorizations) {
    InputStream baseOwlFile = OntologyRepositoryBase.class.getResourceAsStream(fileName);
    checkNotNull(
        baseOwlFile,
        "Could not load resource " + OntologyRepositoryBase.class.getResource(fileName));

    try {
      importFile(baseOwlFile, IRI.create(iri), null, authorizations);
    } catch (Exception e) {
      throw new LumifyException("Could not import ontology file", e);
    } finally {
      IOUtils.closeQuietly(baseOwlFile);
    }
  }

  private void addEntityGlyphIcon(Concept entityConcept) {
    InputStream entityGlyphIconInputStream =
        OntologyRepositoryBase.class.getResourceAsStream("entity.png");
    checkNotNull(
        entityGlyphIconInputStream,
        "Could not load resource " + OntologyRepositoryBase.class.getResource("entity.png"));

    try {
      ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
      IOUtils.copy(entityGlyphIconInputStream, imgOut);

      byte[] rawImg = imgOut.toByteArray();

      addEntityGlyphIconToEntityConcept(entityConcept, rawImg);
    } catch (IOException e) {
      throw new LumifyException("invalid stream for glyph icon");
    }
  }

  protected abstract void addEntityGlyphIconToEntityConcept(Concept entityConcept, byte[] rawImg);

  public boolean isOntologyDefined() {
    try {
      Concept concept = getConceptByIRI(OntologyRepository.ROOT_CONCEPT_IRI);
      return concept != null; // todo should check for more
    } catch (Exception e) {
      if (e.getMessage() != null
          && e.getMessage().contains(OntologyLumifyProperties.ONTOLOGY_TITLE.getPropertyName())) {
        return false;
      }
      throw new RuntimeException(e);
    }
  }

  @Override
  public void importFile(File inFile, IRI documentIRI, Authorizations authorizations)
      throws Exception {
    if (!inFile.exists()) {
      throw new LumifyException("File " + inFile + " does not exist");
    }
    File inDir = inFile.getParentFile();

    FileInputStream inFileIn = new FileInputStream(inFile);
    try {
      importFile(inFileIn, documentIRI, inDir, authorizations);
    } finally {
      inFileIn.close();
    }
  }

  private void importFile(
      InputStream in, IRI documentIRI, File inDir, Authorizations authorizations) throws Exception {
    byte[] inFileData = IOUtils.toByteArray(in);

    Reader inFileReader = new InputStreamReader(new ByteArrayInputStream(inFileData));

    OWLOntologyLoaderConfiguration config = new OWLOntologyLoaderConfiguration();
    OWLOntologyManager m = createOwlOntologyManager(config, documentIRI);

    OWLOntologyDocumentSource documentSource = new ReaderDocumentSource(inFileReader, documentIRI);
    OWLOntology o = m.loadOntologyFromOntologyDocument(documentSource, config);

    storeOntologyFile(new ByteArrayInputStream(inFileData), documentIRI);

    for (OWLClass ontologyClass : o.getClassesInSignature()) {
      if (!o.isDeclared(ontologyClass, false)) {
        continue;
      }
      importOntologyClass(o, ontologyClass, inDir, authorizations);
    }

    for (OWLDataProperty dataTypeProperty : o.getDataPropertiesInSignature()) {
      if (!o.isDeclared(dataTypeProperty, false)) {
        continue;
      }
      importDataProperty(o, dataTypeProperty);
    }

    for (OWLObjectProperty objectProperty : o.getObjectPropertiesInSignature()) {
      if (!o.isDeclared(objectProperty, false)) {
        continue;
      }
      importObjectProperty(o, objectProperty);
    }

    for (OWLObjectProperty objectProperty : o.getObjectPropertiesInSignature()) {
      if (!o.isDeclared(objectProperty, false)) {
        continue;
      }
      importInverseOf(o, objectProperty);
    }
  }

  public OWLOntologyManager createOwlOntologyManager(
      OWLOntologyLoaderConfiguration config, IRI excludeDocumentIRI) throws Exception {
    OWLOntologyManager m = OWLManager.createOWLOntologyManager();
    config.setMissingImportHandlingStrategy(MissingImportHandlingStrategy.SILENT);
    loadOntologyFiles(m, config, excludeDocumentIRI);
    return m;
  }

  protected abstract void storeOntologyFile(InputStream inputStream, IRI documentIRI);

  public void exportOntology(OutputStream out, IRI documentIRI) throws Exception {
    OWLOntologyManager m = OWLManager.createOWLOntologyManager();
    OWLOntologyLoaderConfiguration config = new OWLOntologyLoaderConfiguration();
    config.setMissingImportHandlingStrategy(MissingImportHandlingStrategy.SILENT);

    List<OWLOntology> loadedOntologies = loadOntologyFiles(m, config, null);
    OWLOntology o = findOntology(loadedOntologies, documentIRI);
    if (o == null) {
      throw new LumifyException("Could not find ontology with iri " + documentIRI);
    }

    Writer fileWriter = new OutputStreamWriter(out);

    new RDFXMLRenderer(o, fileWriter).render();
  }

  protected abstract List<OWLOntology> loadOntologyFiles(
      OWLOntologyManager m, OWLOntologyLoaderConfiguration config, IRI excludedIRI)
      throws Exception;

  private OWLOntology findOntology(List<OWLOntology> loadedOntologies, IRI documentIRI) {
    for (OWLOntology o : loadedOntologies) {
      if (documentIRI.equals(o.getOntologyID().getOntologyIRI())) {
        return o;
      }
    }
    return null;
  }

  protected Concept importOntologyClass(
      OWLOntology o, OWLClass ontologyClass, File inDir, Authorizations authorizations)
      throws IOException {
    String uri = ontologyClass.getIRI().toString();
    if ("http://www.w3.org/2002/07/owl#Thing".equals(uri)) {
      return getEntityConcept();
    }

    String label = getLabel(o, ontologyClass);
    checkNotNull(label, "label cannot be null or empty: " + uri);
    LOGGER.info("Importing ontology class " + uri + " (label: " + label + ")");

    Concept parent = getParentConcept(o, ontologyClass, inDir, authorizations);
    Concept result = getOrCreateConcept(parent, uri, label);

    String color = getColor(o, ontologyClass);
    if (color != null) {
      result.setProperty(
          OntologyLumifyProperties.COLOR.getPropertyName(),
          color,
          OntologyRepository.VISIBILITY.getVisibility(),
          authorizations);
    }

    String displayType = getDisplayType(o, ontologyClass);
    if (displayType != null) {
      result.setProperty(
          OntologyLumifyProperties.DISPLAY_TYPE.getPropertyName(),
          displayType,
          OntologyRepository.VISIBILITY.getVisibility(),
          authorizations);
    }

    String titleFormula = getTitleFormula(o, ontologyClass);
    if (titleFormula != null) {
      result.setProperty(
          OntologyLumifyProperties.TITLE_FORMULA.getPropertyName(),
          titleFormula,
          OntologyRepository.VISIBILITY.getVisibility(),
          authorizations);
    }

    String subtitleFormula = getSubtitleFormula(o, ontologyClass);
    if (subtitleFormula != null) {
      result.setProperty(
          OntologyLumifyProperties.SUBTITLE_FORMULA.getPropertyName(),
          subtitleFormula,
          OntologyRepository.VISIBILITY.getVisibility(),
          authorizations);
    }

    String timeFormula = getTimeFormula(o, ontologyClass);
    if (timeFormula != null) {
      result.setProperty(
          OntologyLumifyProperties.TIME_FORMULA.getPropertyName(),
          timeFormula,
          OntologyRepository.VISIBILITY.getVisibility(),
          authorizations);
    }

    boolean userVisible = getUserVisible(o, ontologyClass);
    result.setProperty(
        OntologyLumifyProperties.USER_VISIBLE.getPropertyName(),
        userVisible,
        OntologyRepository.VISIBILITY.getVisibility(),
        authorizations);

    String glyphIconFileName = getGlyphIconFileName(o, ontologyClass);
    setIconProperty(
        result,
        inDir,
        glyphIconFileName,
        LumifyProperties.GLYPH_ICON.getPropertyName(),
        authorizations);

    String mapGlyphIconFileName = getMapGlyphIconFileName(o, ontologyClass);
    setIconProperty(
        result,
        inDir,
        mapGlyphIconFileName,
        LumifyProperties.MAP_GLYPH_ICON.getPropertyName(),
        authorizations);

    return result;
  }

  private void setIconProperty(
      Concept concept,
      File inDir,
      String glyphIconFileName,
      String propertyKey,
      Authorizations authorizations)
      throws IOException {
    if (glyphIconFileName != null) {
      File iconFile = new File(inDir, glyphIconFileName);
      if (!iconFile.exists()) {
        throw new RuntimeException("Could not find icon file: " + iconFile.toString());
      }
      InputStream iconFileIn = new FileInputStream(iconFile);
      try {
        StreamingPropertyValue value = new StreamingPropertyValue(iconFileIn, byte[].class);
        value.searchIndex(false);
        value.store(true);
        concept.setProperty(
            propertyKey, value, OntologyRepository.VISIBILITY.getVisibility(), authorizations);
      } finally {
        iconFileIn.close();
      }
    }
  }

  protected Concept getParentConcept(
      OWLOntology o, OWLClass ontologyClass, File inDir, Authorizations authorizations)
      throws IOException {
    Set<OWLClassExpression> superClasses = ontologyClass.getSuperClasses(o);
    if (superClasses.size() == 0) {
      return getEntityConcept();
    } else if (superClasses.size() == 1) {
      OWLClassExpression superClassExpr = superClasses.iterator().next();
      OWLClass superClass = superClassExpr.asOWLClass();
      String superClassUri = superClass.getIRI().toString();
      Concept parent = getConceptByIRI(superClassUri);
      if (parent != null) {
        return parent;
      }

      parent = importOntologyClass(o, superClass, inDir, authorizations);
      if (parent == null) {
        throw new LumifyException("Could not find or create parent: " + superClass);
      }
      return parent;
    } else {
      throw new LumifyException(
          "Unhandled multiple super classes. Found " + superClasses.size() + ", expected 0 or 1.");
    }
  }

  protected void importDataProperty(OWLOntology o, OWLDataProperty dataTypeProperty) {
    String propertyIRI = dataTypeProperty.getIRI().toString();
    String propertyDisplayName = getLabel(o, dataTypeProperty);
    PropertyType propertyType = getPropertyType(o, dataTypeProperty);
    boolean userVisible = getUserVisible(o, dataTypeProperty);
    boolean searchable = getSearchable(o, dataTypeProperty);
    Boolean displayTime = getDisplayTime(o, dataTypeProperty);
    Double boost = getBoost(o, dataTypeProperty);
    if (propertyType == null) {
      throw new LumifyException("Could not get property type on data property " + propertyIRI);
    }

    for (OWLClassExpression domainClassExpr : dataTypeProperty.getDomains(o)) {
      OWLClass domainClass = domainClassExpr.asOWLClass();
      String domainClassUri = domainClass.getIRI().toString();
      Concept domainConcept = getConceptByIRI(domainClassUri);
      checkNotNull(domainConcept, "Could not find class with uri: " + domainClassUri);

      LOGGER.info("Adding data property " + propertyIRI + " to class " + domainConcept.getTitle());

      ArrayList<PossibleValueType> possibleValues = getPossibleValues(o, dataTypeProperty);
      Collection<TextIndexHint> textIndexHints = getTextIndexHints(o, dataTypeProperty);
      addPropertyTo(
          domainConcept,
          propertyIRI,
          propertyDisplayName,
          propertyType,
          possibleValues,
          textIndexHints,
          userVisible,
          searchable,
          displayTime,
          boost);
    }
  }

  protected abstract OntologyProperty addPropertyTo(
      Concept concept,
      String propertyIRI,
      String displayName,
      PropertyType dataType,
      ArrayList<PossibleValueType> possibleValues,
      Collection<TextIndexHint> textIndexHints,
      boolean userVisible,
      boolean searchable,
      Boolean displayTime,
      Double boost);

  protected void importObjectProperty(OWLOntology o, OWLObjectProperty objectProperty) {
    String iri = objectProperty.getIRI().toString();
    String label = getLabel(o, objectProperty);
    checkNotNull(label, "label cannot be null or empty for " + iri);
    LOGGER.info("Importing ontology object property " + iri + " (label: " + label + ")");

    for (Concept domain : getDomainsConcepts(o, objectProperty)) {
      for (Concept range : getRangesConcepts(o, objectProperty)) {
        getOrCreateRelationshipType(domain, range, iri, label);
      }
    }
  }

  protected void importInverseOf(OWLOntology o, OWLObjectProperty objectProperty) {
    String iri = objectProperty.getIRI().toString();
    Relationship fromRelationship = null;

    for (OWLObjectPropertyExpression inverseOf : objectProperty.getInverses(o)) {
      if (inverseOf instanceof OWLObjectProperty) {
        if (fromRelationship == null) {
          fromRelationship = getRelationshipByIRI(iri);
        }

        OWLObjectProperty inverseOfOWLObjectProperty = (OWLObjectProperty) inverseOf;
        String inverseOfIri = inverseOfOWLObjectProperty.getIRI().toString();
        Relationship inverseOfRelationship = getRelationshipByIRI(inverseOfIri);
        getOrCreateInverseOfRelationship(fromRelationship, inverseOfRelationship);
      }
    }
  }

  protected abstract void getOrCreateInverseOfRelationship(
      Relationship fromRelationship, Relationship inverseOfRelationship);

  private Iterable<Concept> getRangesConcepts(OWLOntology o, OWLObjectProperty objectProperty) {
    String uri = objectProperty.getIRI().toString();
    if (objectProperty.getRanges(o).size() == 0) {
      throw new LumifyException("Invalid number of range properties on " + uri);
    }

    List<Concept> ranges = new ArrayList<Concept>();
    for (OWLClassExpression rangeClassExpr : objectProperty.getRanges(o)) {
      OWLClass rangeClass = rangeClassExpr.asOWLClass();
      String rangeClassUri = rangeClass.getIRI().toString();
      Concept ontologyClass = getConceptByIRI(rangeClassUri);
      checkNotNull(ontologyClass, "Could not find class with uri: " + rangeClassUri);
      ranges.add(ontologyClass);
    }
    return ranges;
  }

  private Iterable<Concept> getDomainsConcepts(OWLOntology o, OWLObjectProperty objectProperty) {
    String uri = objectProperty.getIRI().toString();
    if (objectProperty.getDomains(o).size() == 0) {
      throw new LumifyException("Invalid number of domain properties on " + uri);
    }

    List<Concept> domains = new ArrayList<Concept>();
    for (OWLClassExpression rangeClassExpr : objectProperty.getDomains(o)) {
      OWLClass rangeClass = rangeClassExpr.asOWLClass();
      String rangeClassUri = rangeClass.getIRI().toString();
      Concept ontologyClass = getConceptByIRI(rangeClassUri);
      checkNotNull(ontologyClass, "Could not find class with uri: " + rangeClassUri);
      domains.add(ontologyClass);
    }
    return domains;
  }

  protected PropertyType getPropertyType(OWLOntology o, OWLDataProperty dataTypeProperty) {
    Set<OWLDataRange> ranges = dataTypeProperty.getRanges(o);
    if (ranges.size() == 0) {
      return null;
    }
    if (ranges.size() > 1) {
      throw new LumifyException(
          "Unexpected number of ranges on data property " + dataTypeProperty.getIRI().toString());
    }
    for (OWLDataRange range : ranges) {
      if (range instanceof OWLDatatype) {
        OWLDatatype datatype = (OWLDatatype) range;
        return getPropertyType(datatype.getIRI().toString());
      }
    }
    throw new LumifyException(
        "Could not find range on data property " + dataTypeProperty.getIRI().toString());
  }

  private PropertyType getPropertyType(String iri) {
    if ("http://www.w3.org/2001/XMLSchema#string".equals(iri)) {
      return PropertyType.STRING;
    }
    if ("http://www.w3.org/2001/XMLSchema#dateTime".equals(iri)) {
      return PropertyType.DATE;
    }
    if ("http://www.w3.org/2001/XMLSchema#int".equals(iri)) {
      return PropertyType.DOUBLE;
    }
    if ("http://www.w3.org/2001/XMLSchema#double".equals(iri)) {
      return PropertyType.DOUBLE;
    }
    if ("http://lumify.io#geolocation".equals(iri)) {
      return PropertyType.GEO_LOCATION;
    }
    if ("http://lumify.io#currency".equals(iri)) {
      return PropertyType.CURRENCY;
    }
    if ("http://lumify.io#image".equals(iri)) {
      return PropertyType.IMAGE;
    }
    if ("http://www.w3.org/2001/XMLSchema#hexBinary".equals(iri)) {
      return PropertyType.BINARY;
    }
    if ("http://www.w3.org/2001/XMLSchema#boolean".equals(iri)) {
      return PropertyType.BOOLEAN;
    }
    if ("http://www.w3.org/2001/XMLSchema#integer".equals(iri)) {
      return PropertyType.INTEGER;
    }
    throw new LumifyException("Unhandled property type " + iri);
  }

  public static String getLabel(OWLOntology o, OWLEntity owlEntity) {
    for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
      if (annotation.getProperty().isLabel()) {
        OWLLiteral value = (OWLLiteral) annotation.getValue();
        return value.getLiteral();
      }
    }
    return owlEntity.getIRI().toString();
  }

  protected String getColor(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(o, owlEntity, OntologyLumifyProperties.COLOR.getPropertyName());
  }

  protected String getDisplayType(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.DISPLAY_TYPE.getPropertyName());
  }

  protected String getTitleFormula(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.TITLE_FORMULA.getPropertyName());
  }

  protected String getSubtitleFormula(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.SUBTITLE_FORMULA.getPropertyName());
  }

  protected String getTimeFormula(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.TIME_FORMULA.getPropertyName());
  }

  protected Boolean getDisplayTime(OWLOntology o, OWLEntity owlEntity) {
    String val =
        getAnnotationValueByUri(
            o, owlEntity, OntologyLumifyProperties.DISPLAY_TIME.getPropertyName());
    if (val == null) {
      return null;
    }
    return Boolean.parseBoolean(val);
  }

  protected Double getBoost(OWLOntology o, OWLEntity owlEntity) {
    String val =
        getAnnotationValueByUri(o, owlEntity, OntologyLumifyProperties.BOOST.getPropertyName());
    if (val == null) {
      return null;
    }
    return Double.parseDouble(val);
  }

  protected boolean getUserVisible(OWLOntology o, OWLEntity owlEntity) {
    String val =
        getAnnotationValueByUri(
            o, owlEntity, OntologyLumifyProperties.USER_VISIBLE.getPropertyName());
    return val == null || Boolean.parseBoolean(val);
  }

  protected boolean getSearchable(OWLOntology o, OWLEntity owlEntity) {
    String val = getAnnotationValueByUri(o, owlEntity, "http://lumify.io#searchable");
    return val == null || Boolean.parseBoolean(val);
  }

  protected String getGlyphIconFileName(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.GLYPH_ICON_FILE_NAME.getPropertyName());
  }

  protected String getMapGlyphIconFileName(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValueByUri(
        o, owlEntity, OntologyLumifyProperties.MAP_GLYPH_ICON_FILE_NAME.getPropertyName());
  }

  protected ArrayList<PossibleValueType> getPossibleValues(OWLOntology o, OWLEntity owlEntity) {
    return getAnnotationValuesByUri(
        o, owlEntity, OntologyLumifyProperties.POSSIBLE_VALUES.getPropertyName());
  }

  protected Collection<TextIndexHint> getTextIndexHints(OWLOntology o, OWLDataProperty owlEntity) {
    return TextIndexHint.parse(
        getAnnotationValueByUri(
            o, owlEntity, OntologyLumifyProperties.TEXT_INDEX_HINTS.getPropertyName()));
  }

  protected ArrayList<PossibleValueType> getAnnotationValuesByUri(
      OWLOntology o, OWLEntity owlEntity, String uri) {
    ArrayList<PossibleValueType> possibleValueTypes = new ArrayList<PossibleValueType>();
    for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
      if (annotation.getProperty().getIRI().toString().equals(uri)) {
        OWLLiteral value = (OWLLiteral) annotation.getValue();
        String[] possibleValues = value.getLiteral().split(",");
        for (String possibleValue : possibleValues) {
          String[] parsedValue = possibleValue.split(":");
          checkArgument(
              parsedValue.length == 2, "Possible values must be in the format mapping:value");
          possibleValueTypes.add(new PossibleValueType(parsedValue[0], parsedValue[1]));
        }
      }
    }
    return possibleValueTypes;
  }

  private String getAnnotationValueByUri(OWLOntology o, OWLEntity owlEntity, String uri) {
    for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
      if (annotation.getProperty().getIRI().toString().equals(uri)) {
        OWLLiteral value = (OWLLiteral) annotation.getValue();
        return value.getLiteral();
      }
    }
    return null;
  }

  @Override
  public void writePackage(File file, IRI documentIRI, Authorizations authorizations)
      throws Exception {
    ZipFile zipped = new ZipFile(file);
    if (zipped.isValidZipFile()) {
      File tempDir = Files.createTempDir();
      try {
        LOGGER.info("Extracting: %s to %s", file.getAbsoluteFile(), tempDir.getAbsolutePath());
        zipped.extractAll(tempDir.getAbsolutePath());

        File owlFile = findOwlFile(tempDir);
        importFile(owlFile, documentIRI, authorizations);
      } finally {
        FileUtils.deleteDirectory(tempDir);
      }
    } else {
      importFile(file, documentIRI, authorizations);
    }
  }

  protected File findOwlFile(File dir) {
    File[] files = dir.listFiles();
    if (files == null) {
      return null;
    }
    for (File child : files) {
      if (child.isDirectory()) {
        File found = findOwlFile(child);
        if (found != null) {
          return found;
        }
      } else if (child.getName().toLowerCase().endsWith(".owl")) {
        return child;
      }
    }
    return null;
  }

  @Override
  public void resolvePropertyIds(JSONArray filterJson) throws JSONException {
    for (int i = 0; i < filterJson.length(); i++) {
      JSONObject filter = filterJson.getJSONObject(i);
      if (filter.has("propertyId") && !filter.has("propertyName")) {
        String propertyVertexId = filter.getString("propertyId");
        OntologyProperty property = getProperty(propertyVertexId);
        if (property == null) {
          throw new RuntimeException("Could not find property with id: " + propertyVertexId);
        }
        filter.put("propertyName", property.getTitle());
        filter.put("propertyDataType", property.getDataType());
      }
    }
  }
}