public class ChangeEmail extends BaseRequestHandler {
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ChangeEmail.class);
private static final String EMAIL_PARAMETER_NAME = "email";
@Inject
public ChangeEmail(
UserRepository userRepository,
WorkspaceRepository workspaceRepository,
Configuration configuration) {
super(userRepository, workspaceRepository, configuration);
}
@Override
public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
throws Exception {
User user = getUser(request);
String email = getRequiredParameter(request, EMAIL_PARAMETER_NAME);
if (user != null) {
if (email.length() > 0) {
getUserRepository().setEmailAddress(user, email);
LOGGER.info("changed email for user: %s", user.getUsername());
respondWithSuccessJson(response);
} else {
respondWithBadRequest(response, EMAIL_PARAMETER_NAME, "new email may not be blank");
}
} else {
LOGGER.error("current user not found while attempting to change email");
respondWithAccessDenied(response, "current user not found");
}
}
}
@Override
public void contextInitialized(ServletContextEvent sce) {
final ServletContext context = sce.getServletContext();
LOGGER = LumifyLoggerFactory.getLogger(ApplicationBootstrap.class);
LOGGER.info("Servlet context initialized...");
try {
if (context != null) {
final Configuration config = fetchApplicationConfiguration(context);
LOGGER.info("Running application with configuration:\n%s", config);
InjectHelper.inject(this, LumifyBootstrap.bootstrapModuleMaker(config));
// Store the injector in the context for a servlet to access later
context.setAttribute(Injector.class.getName(), InjectHelper.getInjector());
if (!config.get(Configuration.MODEL_PROVIDER).equals(Configuration.UNKNOWN_STRING)) {
FrameworkUtils.initializeFramework(
InjectHelper.getInjector(), userRepository.getSystemUser());
}
InjectHelper.getInjector().getInstance(OntologyRepository.class);
LOGGER.warn(
"JavaScript / Less modifications will not be reflected on server. Run `grunt watch` from webapp directory in development");
} else {
LOGGER.error("Servlet context could not be acquired!");
}
} catch (Exception ex) {
LOGGER.error("Failed to initialize context", ex);
throw new RuntimeException("Failed to initialize context", ex);
}
}
public class HibernateSessionManager {
private static LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(HibernateSessionManager.class);
private SessionFactory sessionFactory;
private ThreadLocal<Session> threadLocalSession =
new InheritableThreadLocal<Session>() {
@Override
protected Session initialValue() {
LOGGER.info("Opening Hibernate session");
return sessionFactory.openSession();
}
};
// this is to prevent opening a session just to clear it
private ThreadLocal<Boolean> threadLocalSessionIsOpen =
new InheritableThreadLocal<Boolean>() {
@Override
protected Boolean initialValue() {
return Boolean.FALSE;
}
};
public HibernateSessionManager(SessionFactory factory) {
sessionFactory = factory;
threadLocalSession.remove();
}
public Session getSession() {
threadLocalSessionIsOpen.set(Boolean.TRUE);
return threadLocalSession.get();
}
public void clearSession() {
if (threadLocalSessionIsOpen.get()) {
Session session = threadLocalSession.get();
if (session.isOpen()) { // double checking
session.close();
LOGGER.info("Closed Hibernate session");
}
}
threadLocalSession.remove();
threadLocalSessionIsOpen.remove();
}
}
public class FFprobeMakeAndModelUtil {
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(FFprobeMakeAndModelUtil.class);
public static String getMake(JSONObject json) {
return extractMakeAndModelFromJSON(json, "make", "make-eng");
}
public static String getModel(JSONObject json) {
return extractMakeAndModelFromJSON(json, "model", "model-eng");
}
private static String extractMakeAndModelFromJSON(
JSONObject json, String firstSearchString, String secondSearchString) {
if (json == null) {
return null;
}
JSONObject formatObject = json.optJSONObject("format");
if (formatObject != null) {
JSONObject tagsObject = formatObject.optJSONObject("tags");
if (tagsObject != null) {
String firstDesiredValue = tagsObject.optString(firstSearchString);
if (!firstDesiredValue.equals("")) {
return firstDesiredValue;
}
String secondDesiredValue = tagsObject.optString(secondSearchString);
if (!secondDesiredValue.equals("")) {
return secondDesiredValue;
}
}
}
LOGGER.debug(
"Could not extract " + firstSearchString + " or " + secondSearchString + " from json.");
return null;
}
}
public class ResolveTermEntity extends BaseRequestHandler {
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ResolveTermEntity.class);
private static final String MULTI_VALUE_KEY = ResolveTermEntity.class.getName();
private final Graph graph;
private final AuditRepository auditRepository;
private final OntologyRepository ontologyRepository;
private final VisibilityTranslator visibilityTranslator;
private final TermMentionRepository termMentionRepository;
private final WorkspaceRepository workspaceRepository;
private final WorkQueueRepository workQueueRepository;
private final String artifactHasEntityIri;
@Inject
public ResolveTermEntity(
final Graph graphRepository,
final AuditRepository auditRepository,
final OntologyRepository ontologyRepository,
final UserRepository userRepository,
final VisibilityTranslator visibilityTranslator,
final Configuration configuration,
final TermMentionRepository termMentionRepository,
final WorkspaceRepository workspaceRepository,
final WorkQueueRepository workQueueRepository) {
super(userRepository, workspaceRepository, configuration);
this.graph = graphRepository;
this.auditRepository = auditRepository;
this.ontologyRepository = ontologyRepository;
this.visibilityTranslator = visibilityTranslator;
this.termMentionRepository = termMentionRepository;
this.workspaceRepository = workspaceRepository;
this.workQueueRepository = workQueueRepository;
this.artifactHasEntityIri =
this.getConfiguration().get(Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
if (this.artifactHasEntityIri == null) {
throw new LumifyException(
"Could not find configuration for " + Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
}
}
@Override
public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
throws Exception {
final String artifactId = getRequiredParameter(request, "artifactId");
final String propertyKey = getRequiredParameter(request, "propertyKey");
final long mentionStart = getRequiredParameterAsLong(request, "mentionStart");
final long mentionEnd = getRequiredParameterAsLong(request, "mentionEnd");
final String title = getRequiredParameter(request, "sign");
final String conceptId = getRequiredParameter(request, "conceptId");
final String visibilitySource = getRequiredParameter(request, "visibilitySource");
final String resolvedVertexId = getOptionalParameter(request, "resolvedVertexId");
final String justificationText = getOptionalParameter(request, "justificationText");
final String sourceInfo = getOptionalParameter(request, "sourceInfo");
String termMentionRowKeyString = getOptionalParameter(request, "rowKey");
User user = getUser(request);
String workspaceId = getActiveWorkspaceId(request);
Workspace workspace = workspaceRepository.findById(workspaceId, user);
Authorizations authorizations = getAuthorizations(request, user);
JSONObject visibilityJson =
GraphUtil.updateVisibilitySourceAndAddWorkspaceId(null, visibilitySource, workspaceId);
LumifyVisibility visibility = this.visibilityTranslator.toVisibility(visibilityJson);
if (!graph.isVisibilityValid(visibility.getVisibility(), authorizations)) {
LOGGER.warn(
"%s is not a valid visibility for %s user", visibilitySource, user.getDisplayName());
respondWithBadRequest(response, "visibilitySource", STRINGS.getString("visibility.invalid"));
chain.next(request, response);
return;
}
Object id = resolvedVertexId == null ? graph.getIdGenerator().nextId() : resolvedVertexId;
Concept concept = ontologyRepository.getConceptByIRI(conceptId);
final Vertex artifactVertex = graph.getVertex(artifactId, authorizations);
LumifyVisibility lumifyVisibility = visibilityTranslator.toVisibility(visibilityJson);
Map<String, Object> metadata = new HashMap<String, Object>();
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setMetadata(metadata, visibilityJson);
ElementMutation<Vertex> vertexMutation;
Vertex vertex;
if (resolvedVertexId != null) {
vertex = graph.getVertex(id, authorizations);
vertexMutation = vertex.prepareMutation();
} else {
vertexMutation = graph.prepareVertex(id, lumifyVisibility.getVisibility());
GraphUtil.addJustificationToMutation(
vertexMutation, justificationText, sourceInfo, lumifyVisibility);
CONCEPT_TYPE.setProperty(
vertexMutation, conceptId, metadata, lumifyVisibility.getVisibility());
TITLE.addPropertyValue(
vertexMutation, MULTI_VALUE_KEY, title, metadata, lumifyVisibility.getVisibility());
vertex = vertexMutation.save(authorizations);
auditRepository.auditVertexElementMutation(
AuditAction.UPDATE, vertexMutation, vertex, "", user, lumifyVisibility.getVisibility());
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
vertexMutation, visibilityJson, metadata, lumifyVisibility.getVisibility());
this.graph.flush();
workspaceRepository.updateEntityOnWorkspace(
workspace, vertex.getId(), false, null, null, user);
}
// TODO: a better way to check if the same edge exists instead of looking it up every time?
Edge edge =
graph.addEdge(
artifactVertex,
vertex,
this.artifactHasEntityIri,
lumifyVisibility.getVisibility(),
authorizations);
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
edge, visibilityJson, metadata, lumifyVisibility.getVisibility(), authorizations);
auditRepository.auditRelationship(
AuditAction.CREATE,
artifactVertex,
vertex,
edge,
"",
"",
user,
lumifyVisibility.getVisibility());
TermMentionRowKey termMentionRowKey;
if (termMentionRowKeyString != null) {
termMentionRowKey = new TermMentionRowKey(RowKeyHelper.jsonDecode(termMentionRowKeyString));
} else {
termMentionRowKey =
new TermMentionRowKey(
artifactId, propertyKey, mentionStart, mentionEnd, edge.getId().toString());
}
TermMentionModel termMention = new TermMentionModel(termMentionRowKey);
termMention
.getMetadata()
.setSign(title, lumifyVisibility.getVisibility())
.setOntologyClassUri(concept.getDisplayName(), lumifyVisibility.getVisibility())
.setConceptGraphVertexId(concept.getTitle(), lumifyVisibility.getVisibility())
.setVertexId(vertex.getId().toString(), lumifyVisibility.getVisibility())
.setEdgeId(edge.getId().toString(), lumifyVisibility.getVisibility());
termMentionRepository.save(termMention);
vertexMutation.addPropertyValue(
graph.getIdGenerator().nextId().toString(),
LumifyProperties.ROW_KEY.getPropertyName(),
termMentionRowKey.toString(),
metadata,
lumifyVisibility.getVisibility());
vertexMutation.save(authorizations);
this.graph.flush();
workQueueRepository.pushTextUpdated(artifactId);
workQueueRepository.pushElement(edge);
JSONObject result = new JSONObject();
result.put("success", true);
respondWithJson(response, result);
}
}
public class RelationshipSetVisibility extends BaseRequestHandler {
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(RelationshipSetVisibility.class);
private final Graph graph;
private final VisibilityTranslator visibilityTranslator;
private final WorkQueueRepository workQueueRepository;
private final AuditRepository auditRepository;
@Inject
public RelationshipSetVisibility(
final Graph graph,
final UserRepository userRepository,
final Configuration configuration,
final VisibilityTranslator visibilityTranslator,
final WorkspaceRepository workspaceRepository,
final WorkQueueRepository workQueueRepository,
final AuditRepository auditRepository) {
super(userRepository, workspaceRepository, configuration);
this.graph = graph;
this.visibilityTranslator = visibilityTranslator;
this.workQueueRepository = workQueueRepository;
this.auditRepository = auditRepository;
}
@Override
public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
throws Exception {
final String graphEdgeId = getAttributeString(request, "graphEdgeId");
final String visibilitySource = getRequiredParameter(request, "visibilitySource");
User user = getUser(request);
Authorizations authorizations = getAuthorizations(request, user);
String workspaceId = getActiveWorkspaceId(request);
Edge graphEdge = graph.getEdge(graphEdgeId, authorizations);
if (graphEdge == null) {
respondWithNotFound(response);
return;
}
if (!graph.isVisibilityValid(new Visibility(visibilitySource), authorizations)) {
LOGGER.warn(
"%s is not a valid visibility for %s user", visibilitySource, user.getDisplayName());
respondWithBadRequest(response, "visibilitySource", getString(request, "visibility.invalid"));
chain.next(request, response);
return;
}
LOGGER.info(
"changing edge (%s) visibility source to %s",
graphEdge.getId().toString(), visibilitySource);
GraphUtil.VisibilityAndElementMutation<Edge> setPropertyResult =
GraphUtil.updateElementVisibilitySource(
visibilityTranslator,
graphEdge,
GraphUtil.getSandboxStatus(graphEdge, workspaceId),
visibilitySource,
workspaceId,
authorizations);
auditRepository.auditEdgeElementMutation(
AuditAction.UPDATE,
setPropertyResult.elementMutation,
graphEdge,
graphEdge.getVertex(Direction.OUT, authorizations),
graphEdge.getVertex(Direction.IN, authorizations),
"",
user,
setPropertyResult.visibility.getVisibility());
this.graph.flush();
this.workQueueRepository.pushGraphPropertyQueue(
graphEdge,
null,
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.getPropertyName(),
workspaceId,
visibilitySource);
JSONObject json = JsonSerializer.toJson(graphEdge, workspaceId, authorizations);
respondWithJson(response, json);
}
}
public class ImportMR extends LumifyMRBase {
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ImportMR.class);
public static final String WIKIPEDIA_MIME_TYPE = "text/plain";
public static final String WIKIPEDIA_SOURCE = "Wikipedia";
public static final String WIKIPEDIA_ID_PREFIX = "WIKIPEDIA_";
public static final String WIKIPEDIA_LINK_ID_PREFIX = "WIKIPEDIA_LINK_";
public static final String MULTI_VALUE_KEY = ImportMR.class.getName();
private OntologyRepository ontologyRepository;
static String getWikipediaPageVertexId(String pageTitle) {
return WIKIPEDIA_ID_PREFIX + pageTitle.trim().toLowerCase();
}
static String getWikipediaPageToPageEdgeId(Vertex pageVertex, Vertex linkedPageVertex) {
return WIKIPEDIA_LINK_ID_PREFIX
+ getWikipediaPageTitleFromId(pageVertex.getId())
+ "_"
+ getWikipediaPageTitleFromId(linkedPageVertex.getId());
}
static String getWikipediaPageTitleFromId(Object id) {
return id.toString().substring(WIKIPEDIA_ID_PREFIX.length());
}
@Override
protected void setupJob(Job job) throws Exception {
verifyWikipediaPageConcept(ontologyRepository);
verifyWikipediaPageInternalLinkWikipediaPageRelationship(ontologyRepository);
job.setJarByClass(ImportMR.class);
job.setMapperClass(ImportMRMapper.class);
job.setNumReduceTasks(0);
job.setMapOutputValueClass(Mutation.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(AccumuloElementOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(getConf().get("in")));
}
@Override
protected String getJobName() {
return "wikipediaImport";
}
private void verifyWikipediaPageInternalLinkWikipediaPageRelationship(
OntologyRepository ontologyRepository) {
Relationship wikipediaPageInternalLinkWikipediaPageRelationship =
ontologyRepository.getRelationshipByIRI(
WikipediaConstants.WIKIPEDIA_PAGE_INTERNAL_LINK_WIKIPEDIA_PAGE_CONCEPT_URI);
if (wikipediaPageInternalLinkWikipediaPageRelationship == null) {
throw new RuntimeException(
WikipediaConstants.WIKIPEDIA_PAGE_INTERNAL_LINK_WIKIPEDIA_PAGE_CONCEPT_URI
+ " relationship not found");
}
}
private void verifyWikipediaPageConcept(OntologyRepository ontologyRepository) {
Concept wikipediaPageConcept =
ontologyRepository.getConceptByIRI(WikipediaConstants.WIKIPEDIA_PAGE_CONCEPT_URI);
if (wikipediaPageConcept == null) {
throw new RuntimeException(
WikipediaConstants.WIKIPEDIA_PAGE_CONCEPT_URI + " concept not found");
}
}
@Override
protected void parseArgs(JobConf conf, String[] args) {
if (args.length != 1) {
throw new RuntimeException("Required arguments <inputFileName>");
}
String inFileName = args[0];
LOGGER.info("inFileName: %s", inFileName);
conf.set("in", inFileName);
conf.set(ImportMRMapper.CONFIG_SOURCE_FILE_NAME, new File(inFileName).getName());
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new ImportMR(), args);
System.exit(res);
}
@Inject
public void setOntologyRepository(OntologyRepository ontologyRepository) {
this.ontologyRepository = ontologyRepository;
}
}
public class ArtifactRaw extends BaseRequestHandler {
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ArtifactRaw.class);
private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=([0-9]*)-([0-9]*)");
private final Graph graph;
@Inject
public ArtifactRaw(
final Graph graph,
final UserRepository userRepository,
final WorkspaceRepository workspaceRepository,
final Configuration configuration) {
super(userRepository, workspaceRepository, configuration);
this.graph = graph;
}
@Override
public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
throws Exception {
boolean download = getOptionalParameter(request, "download") != null;
boolean playback = getOptionalParameter(request, "playback") != null;
User user = getUser(request);
Authorizations authorizations = getAuthorizations(request, user);
String graphVertexId = UrlUtils.urlDecode(getAttributeString(request, "graphVertexId"));
Vertex artifactVertex = graph.getVertex(graphVertexId, authorizations);
if (artifactVertex == null) {
respondWithNotFound(response);
return;
}
String fileName = LumifyProperties.FILE_NAME.getPropertyValue(artifactVertex);
if (fileName == null) {
fileName = LumifyProperties.TITLE.getPropertyValue(artifactVertex);
}
if (playback) {
handlePartialPlayback(request, response, artifactVertex, fileName, user);
} else {
String mimeType = getMimeType(artifactVertex);
response.setContentType(mimeType);
setMaxAge(response, EXPIRES_1_HOUR);
if (download) {
response.addHeader("Content-Disposition", "attachment; filename=" + fileName);
} else {
response.addHeader("Content-Disposition", "inline; filename=" + fileName);
}
StreamingPropertyValue rawValue = LumifyProperties.RAW.getPropertyValue(artifactVertex);
if (rawValue == null) {
LOGGER.warn("Could not find raw on artifact: %s", artifactVertex.getId().toString());
respondWithNotFound(response);
return;
}
InputStream in = rawValue.getInputStream();
try {
IOUtils.copy(in, response.getOutputStream());
} finally {
in.close();
}
}
chain.next(request, response);
}
private void handlePartialPlayback(
HttpServletRequest request,
HttpServletResponse response,
Vertex artifactVertex,
String fileName,
User user)
throws IOException {
String type = getRequiredParameter(request, "type");
InputStream in;
Long totalLength = null;
long partialStart = 0;
Long partialEnd = null;
String range = request.getHeader("Range");
if (range != null) {
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
Matcher m = RANGE_PATTERN.matcher(range);
if (m.matches()) {
partialStart = Long.parseLong(m.group(1));
if (m.group(2).length() > 0) {
partialEnd = Long.parseLong(m.group(2));
}
}
}
response.setCharacterEncoding(null);
response.setContentType(type);
response.addHeader("Content-Disposition", "attachment; filename=" + fileName);
StreamingPropertyValue mediaPropertyValue = getStreamingPropertyValue(artifactVertex, type);
totalLength = mediaPropertyValue.getLength();
in = mediaPropertyValue.getInputStream();
if (partialEnd == null) {
partialEnd = totalLength;
}
// Ensure that the last byte position is less than the instance-length
partialEnd = Math.min(partialEnd, totalLength - 1);
long partialLength = totalLength;
if (range != null) {
partialLength = partialEnd - partialStart + 1;
response.addHeader(
"Content-Range", "bytes " + partialStart + "-" + partialEnd + "/" + totalLength);
if (partialStart > 0) {
in.skip(partialStart);
}
}
response.addHeader("Content-Length", "" + partialLength);
OutputStream out = response.getOutputStream();
copy(in, out, partialLength);
response.flushBuffer();
}
private StreamingPropertyValue getStreamingPropertyValue(Vertex artifactVertex, String type) {
StreamingPropertyValue mediaPropertyValue;
if (MediaLumifyProperties.MIME_TYPE_AUDIO_MP4.equals(type)) {
mediaPropertyValue = MediaLumifyProperties.AUDIO_MP4.getPropertyValue(artifactVertex);
checkNotNull(
mediaPropertyValue,
String.format(
"Could not find %s property on artifact %s",
MediaLumifyProperties.AUDIO_MP4, artifactVertex.getId()));
} else if (MediaLumifyProperties.MIME_TYPE_AUDIO_OGG.equals(type)) {
mediaPropertyValue = MediaLumifyProperties.AUDIO_OGG.getPropertyValue(artifactVertex);
checkNotNull(
mediaPropertyValue,
String.format(
"Could not find %s property on artifact %s",
MediaLumifyProperties.AUDIO_OGG, artifactVertex.getId()));
} else if (MediaLumifyProperties.MIME_TYPE_VIDEO_MP4.equals(type)) {
mediaPropertyValue = MediaLumifyProperties.VIDEO_MP4.getPropertyValue(artifactVertex);
checkNotNull(
mediaPropertyValue,
String.format(
"Could not find %s property on artifact %s",
MediaLumifyProperties.VIDEO_MP4, artifactVertex.getId()));
} else if (MediaLumifyProperties.MIME_TYPE_VIDEO_WEBM.equals(type)) {
mediaPropertyValue = MediaLumifyProperties.VIDEO_WEBM.getPropertyValue(artifactVertex);
checkNotNull(
mediaPropertyValue,
String.format(
"Could not find %s property on artifact %s",
MediaLumifyProperties.VIDEO_WEBM, artifactVertex.getId()));
} else {
throw new RuntimeException("Invalid video type: " + type);
}
return mediaPropertyValue;
}
private void copy(InputStream in, OutputStream out, Long length) throws IOException {
byte[] buffer = new byte[1024];
int read = 0;
while (length > 0 && (read = in.read(buffer, 0, (int) Math.min(length, buffer.length))) > 0) {
out.write(buffer, 0, read);
length -= read;
}
}
private String getMimeType(Vertex artifactVertex) {
String mimeType = LumifyProperties.MIME_TYPE.getPropertyValue(artifactVertex);
if (mimeType == null || mimeType.isEmpty()) {
mimeType = "application/octet-stream";
}
return mimeType;
}
}
public class GraphVertexSearch extends BaseRequestHandler {
private final int DEFAULT_RESULT_COUNT = 100;
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(GraphVertexSearch.class);
private final Graph graph;
private final OntologyRepository ontologyRepository;
private final DetectedObjectRepository detectedObjectRepository;
private final UserRepository userRepository;
@Inject
public GraphVertexSearch(
final OntologyRepository ontologyRepository,
final Graph graph,
final UserRepository userRepository,
final Configuration configuration,
final WorkspaceRepository workspaceRepository,
final DetectedObjectRepository detectedObjectRepository) {
super(userRepository, workspaceRepository, configuration);
this.ontologyRepository = ontologyRepository;
this.graph = graph;
this.detectedObjectRepository = detectedObjectRepository;
this.userRepository = userRepository;
}
@Override
public void handle(HttpServletRequest request, HttpServletResponse response, HandlerChain chain)
throws Exception {
long totalStartTime = System.nanoTime();
final String query;
final String filter = getRequiredParameter(request, "filter");
final int offset = (int) getOptionalParameterLong(request, "offset", 0);
final int size = (int) getOptionalParameterLong(request, "size", DEFAULT_RESULT_COUNT);
final String conceptType = getOptionalParameter(request, "conceptType");
final String getLeafNodes = getOptionalParameter(request, "leafNodes");
final String relatedToVertexId = getOptionalParameter(request, "relatedToVertexId");
if (relatedToVertexId == null) {
query = getRequiredParameter(request, "q");
} else {
query = getOptionalParameter(request, "q");
}
long startTime = System.nanoTime();
User user = getUser(request);
Authorizations authorizations = getAuthorizations(request, user);
String workspaceId = getActiveWorkspaceId(request);
ModelUserContext modelUserContext =
userRepository.getModelUserContext(authorizations, workspaceId);
JSONArray filterJson = new JSONArray(filter);
ontologyRepository.resolvePropertyIds(filterJson);
graph.flush();
LOGGER.debug("search %s\n%s", query, filterJson.toString(2));
Query graphQuery;
if (relatedToVertexId == null) {
graphQuery = graph.query(query, authorizations);
} else if (query == null || StringUtils.isBlank(query)) {
graphQuery = graph.getVertex(relatedToVertexId, authorizations).query(authorizations);
} else {
graphQuery = graph.getVertex(relatedToVertexId, authorizations).query(query, authorizations);
}
for (int i = 0; i < filterJson.length(); i++) {
JSONObject obj = filterJson.getJSONObject(i);
if (obj.length() > 0) {
updateQueryWithFilter(graphQuery, obj);
}
}
if (conceptType != null) {
Concept concept = ontologyRepository.getConceptByIRI(conceptType);
if (getLeafNodes == null || !getLeafNodes.equals("false")) {
List<Concept> leafNodeList = ontologyRepository.getAllLeafNodesByConcept(concept);
if (leafNodeList.size() > 0) {
String[] conceptIds = new String[leafNodeList.size()];
int count = 0;
for (Concept c : leafNodeList) {
conceptIds[count] = c.getTitle();
count++;
}
graphQuery.has(CONCEPT_TYPE.getPropertyName(), Compare.IN, conceptIds);
}
} else {
graphQuery.has(CONCEPT_TYPE.getPropertyName(), conceptType);
}
}
graphQuery.limit(size);
graphQuery.skip(offset);
Iterable<Vertex> searchResults;
try {
searchResults = graphQuery.vertices();
} catch (SearchPhaseExecutionException ex) {
respondWithBadRequest(response, "q", "Invalid Query");
return;
}
Map<Object, Double> scores = null;
if (searchResults instanceof IterableWithScores) {
scores = ((IterableWithScores) searchResults).getScores();
}
long retrievalStartTime = System.nanoTime();
List<JSONObject> verticesJsonList = new ArrayList<JSONObject>();
for (Vertex vertex : searchResults) {
JSONObject vertexJson = JsonSerializer.toJson(vertex, workspaceId);
if (scores != null) {
vertexJson.put("score", scores.get(vertex.getId()));
}
vertexJson.put(
"detectedObjects",
detectedObjectRepository.toJSON(vertex, modelUserContext, authorizations, workspaceId));
verticesJsonList.add(vertexJson);
}
long retrievalEndTime = System.nanoTime();
Collections.sort(
verticesJsonList,
new Comparator<JSONObject>() {
@Override
public int compare(JSONObject o1, JSONObject o2) {
double score1 = o1.optDouble("score", 0.0);
double score2 = o2.optDouble("score", 0.0);
return -Double.compare(score1, score2);
}
});
JSONArray verticesJson = new JSONArray();
for (JSONObject vertexJson : verticesJsonList) {
verticesJson.put(vertexJson);
}
long totalEndTime = System.nanoTime();
JSONObject results = new JSONObject();
results.put("vertices", verticesJson);
results.put("nextOffset", offset + size);
results.put("retrievalTime", retrievalEndTime - retrievalStartTime);
results.put("totalTime", totalEndTime - totalStartTime);
if (searchResults instanceof IterableWithTotalHits) {
results.put("totalHits", ((IterableWithTotalHits) searchResults).getTotalHits());
}
if (searchResults instanceof IterableWithSearchTime) {
results.put(
"searchTime", ((IterableWithSearchTime) searchResults).getSearchTimeNanoSeconds());
}
long endTime = System.nanoTime();
LOGGER.info(
"Search for \"%s\" found %d vertices in %dms",
query, verticesJsonList.size(), (endTime - startTime) / 1000 / 1000);
respondWithJson(response, results);
}
private void updateQueryWithFilter(Query graphQuery, JSONObject obj) throws ParseException {
String predicateString = obj.optString("predicate");
JSONArray values = obj.getJSONArray("values");
PropertyType propertyDataType = PropertyType.convert(obj.optString("propertyDataType"));
String propertyName = obj.getString("propertyName");
Object value0 = jsonValueToObject(values, propertyDataType, 0);
if (PropertyType.STRING.equals(propertyDataType)
&& (predicateString == null || "".equals(predicateString))) {
graphQuery.has(propertyName, TextPredicate.CONTAINS, value0);
} else if (PropertyType.BOOLEAN.equals(propertyDataType)
&& (predicateString == null || "".equals(predicateString))) {
graphQuery.has(propertyName, Compare.EQUAL, value0);
} else if ("<".equals(predicateString)) {
graphQuery.has(propertyName, Compare.LESS_THAN, value0);
} else if (">".equals(predicateString)) {
graphQuery.has(propertyName, Compare.GREATER_THAN, value0);
} else if ("range".equals(predicateString)) {
graphQuery.has(propertyName, Compare.GREATER_THAN_EQUAL, value0);
graphQuery.has(
propertyName, Compare.LESS_THAN_EQUAL, jsonValueToObject(values, propertyDataType, 1));
} else if ("=".equals(predicateString) || "equal".equals(predicateString)) {
graphQuery.has(propertyName, Compare.EQUAL, value0);
} else if (PropertyType.GEO_LOCATION.equals(propertyDataType)) {
graphQuery.has(propertyName, GeoCompare.WITHIN, value0);
} else {
throw new LumifyException("unhandled query\n" + obj.toString(2));
}
}
private Object jsonValueToObject(JSONArray values, PropertyType propertyDataType, int index)
throws ParseException {
return OntologyProperty.convert(values, propertyDataType, index);
}
}
public abstract class RegexGraphPropertyWorker extends GraphPropertyWorker {
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(RegexGraphPropertyWorker.class);
private final Pattern pattern;
public RegexGraphPropertyWorker(String regEx) {
this.pattern = Pattern.compile(regEx, Pattern.MULTILINE);
}
protected abstract String getOntologyClassUri();
@Override
public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
super.prepare(workerPrepareData);
LOGGER.debug(
"Extractor prepared for entity type [%s] with regular expression: %s",
getOntologyClassUri(), this.pattern.toString());
}
@Override
public void execute(InputStream in, GraphPropertyWorkData data) throws Exception {
LOGGER.debug("Extracting pattern [%s] from provided text", pattern);
final String text = CharStreams.toString(new InputStreamReader(in, Charsets.UTF_8));
final Matcher matcher = pattern.matcher(text);
List<TermMention> termMentions = new ArrayList<TermMention>();
while (matcher.find()) {
TermMention termMention =
createTerm(matcher, data.getProperty().getKey(), data.getVisibility());
termMentions.add(termMention);
}
Vertex v = (Vertex) data.getElement();
getAuditRepository()
.auditAnalyzedBy(
AuditAction.ANALYZED_BY, v, getClass().getSimpleName(), getUser(), v.getVisibility());
saveTermMentions((Vertex) data.getElement(), termMentions);
}
private TermMention createTerm(final Matcher matched, String propertyKey, Visibility visibility) {
final String patternGroup = matched.group();
int start = matched.start();
int end = matched.end();
return new TermMention.Builder(
start, end, patternGroup, getOntologyClassUri(), propertyKey, visibility)
.resolved(false)
.useExisting(true)
.process(getClass().getName())
.build();
}
@Override
public boolean isHandled(Element element, Property property) {
if (property == null) {
return false;
}
if (property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
return false;
}
String mimeType =
(String) property.getMetadata().get(LumifyProperties.MIME_TYPE.getPropertyName());
return !(mimeType == null || !mimeType.startsWith("text"));
}
}
@Parameters(
separators = "=",
commandDescription = "Exports Palantir Oracle database to hadoop SequenceFiles")
public class DataToSequenceFile implements Exporter.ExporterSource {
public static final String ONTOLOGY_XML_DIR_NAME = "OntologyXML";
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(DataToSequenceFile.class);
@Parameter(
names = {"-n", "--namespace"},
description = "Table namespace",
required = true)
private String tableNamespace;
@Parameter(
names = {"-c", "--connectionstring"},
description = "Database connection string",
required = true)
private String databaseConnectionString;
@Parameter(
names = {"-u", "--username"},
description = "Database username",
required = true)
private String databaseUsername;
@Parameter(
names = {"-p", "--password"},
description = "Database password",
required = true)
private String databasePassword;
@Parameter(
names = {"-d", "--dest"},
description = "Destination path hadoop url",
required = true)
private String destination;
@Parameter(
names = {"--baseiri"},
description = "base IRI for ontology",
required = true)
private String baseIri;
@Parameter(
names = {"--exporters"},
description = "comma separated list of exporters (default: all)")
private String exporters = "all";
private Path destinationPath;
private FileSystem fs;
private SqlRunner sqlRunner;
private Configuration hadoopConfiguration;
public static final List<Exporter> EXPORTERS = new ArrayList<>();
static {
EXPORTERS.add(new PtObjectTypeExporter());
EXPORTERS.add(new PtPropertyTypeExporter());
EXPORTERS.add(new PtLinkTypeExporter());
EXPORTERS.add(new PtTypeGroupExporter());
EXPORTERS.add(new PtNodeDisplayTypeExporter());
EXPORTERS.add(new PtImageInfoExporter());
EXPORTERS.add(new PtOntologyResourceExporter());
EXPORTERS.add(new PtLinkRelationExporter());
EXPORTERS.add(new PtUserExporter());
EXPORTERS.add(new PtGraphExporter());
EXPORTERS.add(new PtObjectExporter());
EXPORTERS.add(new PtGraphObjectExporter());
EXPORTERS.add(new PtObjectObjectExporter());
EXPORTERS.add(new PtMediaAndValueExporter());
EXPORTERS.add(new PtPropertyAndValueExporter());
EXPORTERS.add(new PtNoteAndNoteValueExporter());
}
public DataToSequenceFile(String[] args) {
new JCommander(this, args);
EXPORTERS.add(new OntologyToOwl(baseIri));
if (exporters.equalsIgnoreCase("all")) {
exporters =
Joiner.on(',')
.join(
new ConvertingIterable<Exporter, String>(EXPORTERS) {
@Override
protected String convert(Exporter o) {
return o.getObjectClass().getSimpleName();
}
});
}
}
public static void main(String[] args) throws Exception {
try {
new DataToSequenceFile(args).run();
} catch (Throwable ex) {
LOGGER.error("Failed to export", ex);
}
}
private void run() throws Exception {
LOGGER.info("begin export");
List<Exporter> exportersToRun = getExportersToRun(exporters);
for (Exporter exporter : exportersToRun) {
LOGGER.info("Preparing to run: %s", exporter.getClass().getSimpleName());
}
hadoopConfiguration = new Configuration(true);
destinationPath = new Path(destination);
fs = getFileSystem(hadoopConfiguration);
sqlRunner = createSqlRunner();
sqlRunner.connect();
try {
for (Exporter exporter : exportersToRun) {
LOGGER.info("Running: %s", exporter.getClass().getSimpleName());
exporter.run(this);
}
} finally {
sqlRunner.close();
}
LOGGER.info("export complete!");
}
private List<Exporter> getExportersToRun(String exporters) {
List<Exporter> results = new ArrayList<>();
for (String exporterString : exporters.split(",")) {
Exporter e = null;
for (Exporter exporter : EXPORTERS) {
if (exporter.getObjectClass().getSimpleName().equalsIgnoreCase(exporterString)) {
e = exporter;
break;
}
}
if (e == null) {
throw new RuntimeException("invalid exporter: " + exporterString);
}
results.add(e);
}
return results;
}
private SqlRunner createSqlRunner() {
return new SqlRunner(
databaseConnectionString, databaseUsername, databasePassword, tableNamespace);
}
private FileSystem getFileSystem(Configuration hadoopConfiguration)
throws IOException, URISyntaxException {
FileSystem fs = FileSystem.get(new URI(destination), hadoopConfiguration);
fs.mkdirs(new Path(destination));
return fs;
}
public Path getDestinationPath() {
return destinationPath;
}
public SqlRunner getSqlRunner() {
return sqlRunner;
}
@Override
public FileSystem getFileSystem() {
return fs;
}
public Configuration getHadoopConfiguration() {
return hadoopConfiguration;
}
public void writeFile(String fileName, String data) {
Path path = new Path(getDestinationPath(), fileName);
try {
try (FSDataOutputStream out = getFileSystem().create(path, true)) {
out.write(data.getBytes());
}
} catch (IOException e) {
throw new RuntimeException("Could not write file: " + path, e);
}
}
}
public abstract class GraphPropertyWorker {
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(GraphPropertyWorker.class);
private Graph graph;
private WorkQueueRepository workQueueRepository;
private WorkspaceRepository workspaceRepository;
private OntologyRepository ontologyRepository;
private AuditRepository auditRepository;
private TermMentionRepository termMentionRepository;
private GraphPropertyWorkerPrepareData workerPrepareData;
private Configuration configuration;
private String artifactHasEntityIri;
private String locationIri;
private String organizationIri;
private String personIri;
private VisibilityTranslator visibilityTranslator;
public void prepare(GraphPropertyWorkerPrepareData workerPrepareData) throws Exception {
this.workerPrepareData = workerPrepareData;
this.artifactHasEntityIri =
getConfiguration().get(Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
if (this.artifactHasEntityIri == null) {
throw new LumifyException(
"Could not find configuration for " + Configuration.ONTOLOGY_IRI_ARTIFACT_HAS_ENTITY);
}
this.locationIri =
(String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_LOCATION);
if (this.locationIri == null || this.locationIri.length() == 0) {
throw new LumifyException(
"Could not find configuration: " + Configuration.ONTOLOGY_IRI_LOCATION);
}
this.organizationIri =
(String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_ORGANIZATION);
if (this.organizationIri == null || this.organizationIri.length() == 0) {
throw new LumifyException(
"Could not find configuration: " + Configuration.ONTOLOGY_IRI_ORGANIZATION);
}
this.personIri =
(String) workerPrepareData.getStormConf().get(Configuration.ONTOLOGY_IRI_PERSON);
if (this.personIri == null || this.personIri.length() == 0) {
throw new LumifyException(
"Could not find configuration: " + Configuration.ONTOLOGY_IRI_PERSON);
}
}
public abstract void execute(InputStream in, GraphPropertyWorkData data) throws Exception;
public abstract boolean isHandled(Element element, Property property);
public boolean isLocalFileRequired() {
return false;
}
protected User getUser() {
return this.workerPrepareData.getUser();
}
public Authorizations getAuthorizations() {
return this.workerPrepareData.getAuthorizations();
}
@Inject
public final void setGraph(Graph graph) {
this.graph = graph;
}
protected Graph getGraph() {
return graph;
}
@Inject
public final void setWorkQueueRepository(WorkQueueRepository workQueueRepository) {
this.workQueueRepository = workQueueRepository;
}
@Inject
public final void setWorkspaceRepository(WorkspaceRepository workspaceRepository) {
this.workspaceRepository = workspaceRepository;
}
protected WorkQueueRepository getWorkQueueRepository() {
return workQueueRepository;
}
protected OntologyRepository getOntologyRepository() {
return ontologyRepository;
}
@Inject
public final void setOntologyRepository(OntologyRepository ontologyRepository) {
this.ontologyRepository = ontologyRepository;
}
protected AuditRepository getAuditRepository() {
return auditRepository;
}
@Inject
public final void setAuditRepository(AuditRepository auditRepository) {
this.auditRepository = auditRepository;
}
@Inject
public final void setVisibilityTranslatory(VisibilityTranslator visibilityTranslator) {
this.visibilityTranslator = visibilityTranslator;
}
protected TermMentionRepository getTermMentionRepository() {
return termMentionRepository;
}
@Inject
public void setTermMentionRepository(TermMentionRepository termMentionRepository) {
this.termMentionRepository = termMentionRepository;
}
protected Configuration getConfiguration() {
return configuration;
}
@Inject
public final void setConfiguration(Configuration configuration) {
this.configuration = configuration;
}
/** Determines if this is a property that should be analyzed by text processing tools. */
protected boolean isTextProperty(Property property) {
if (property == null) {
return false;
}
if (property.getName().equals(LumifyProperties.RAW.getPropertyName())) {
return false;
}
String mimeType =
(String) property.getMetadata().get(LumifyProperties.MIME_TYPE.getPropertyName());
return !(mimeType == null || !mimeType.startsWith("text"));
}
protected String mapToOntologyIri(String type) {
String ontologyClassUri;
if ("location".equals(type)) {
ontologyClassUri = this.locationIri;
} else if ("organization".equals(type)) {
ontologyClassUri = this.organizationIri;
} else if ("person".equals(type)) {
ontologyClassUri = this.personIri;
} else {
ontologyClassUri = LumifyProperties.CONCEPT_TYPE_THING;
}
return ontologyClassUri;
}
protected void addVideoTranscriptAsTextPropertiesToMutation(
ExistingElementMutation<Vertex> mutation,
String propertyKey,
VideoTranscript videoTranscript,
Map<String, Object> metadata,
Visibility visibility) {
metadata.put(LumifyProperties.META_DATA_MIME_TYPE, "text/plain");
for (VideoTranscript.TimedText entry : videoTranscript.getEntries()) {
String textPropertyKey = getVideoTranscriptTimedTextPropertyKey(propertyKey, entry);
StreamingPropertyValue value =
new StreamingPropertyValue(
new ByteArrayInputStream(entry.getText().getBytes()), String.class);
LumifyProperties.TEXT.addPropertyValue(
mutation, textPropertyKey, value, metadata, visibility);
}
}
protected void pushVideoTranscriptTextPropertiesOnWorkQueue(
Element element, String propertyKey, VideoTranscript videoTranscript) {
for (VideoTranscript.TimedText entry : videoTranscript.getEntries()) {
String textPropertyKey = getVideoTranscriptTimedTextPropertyKey(propertyKey, entry);
getWorkQueueRepository()
.pushGraphPropertyQueue(
element, textPropertyKey, LumifyProperties.TEXT.getPropertyName());
}
}
private String getVideoTranscriptTimedTextPropertyKey(
String propertyKey, VideoTranscript.TimedText entry) {
String startTime = String.format("%08d", Math.max(0L, entry.getTime().getStart()));
String endTime = String.format("%08d", Math.max(0L, entry.getTime().getEnd()));
return propertyKey
+ RowKeyHelper.MINOR_FIELD_SEPARATOR
+ MediaLumifyProperties.VIDEO_FRAME.getPropertyName()
+ RowKeyHelper.MINOR_FIELD_SEPARATOR
+ startTime
+ RowKeyHelper.MINOR_FIELD_SEPARATOR
+ endTime;
}
protected void saveTermExtractionResult(
Vertex artifactGraphVertex,
TermExtractionResult termExtractionResult,
String workspaceId,
String visibilitySource) {
List<TermMentionWithGraphVertex> termMentionsResults =
saveTermMentions(
artifactGraphVertex,
termExtractionResult.getTermMentions(),
workspaceId,
visibilitySource);
saveRelationships(termExtractionResult.getRelationships(), termMentionsResults);
saveVertexRelationships(termExtractionResult.getVertexRelationships(), termMentionsResults);
}
private void saveVertexRelationships(
List<VertexRelationship> vertexRelationships,
List<TermMentionWithGraphVertex> termMentionsWithGraphVertices) {
for (VertexRelationship vertexRelationship : vertexRelationships) {
TermMentionWithGraphVertex sourceTermMentionsWithGraphVertex =
findTermMentionWithGraphVertex(
termMentionsWithGraphVertices, vertexRelationship.getSource());
checkNotNull(sourceTermMentionsWithGraphVertex, "Could not find source");
checkNotNull(sourceTermMentionsWithGraphVertex.getVertex(), "Could not find source vertex");
Vertex targetVertex = graph.getVertex(vertexRelationship.getTargetId(), getAuthorizations());
checkNotNull(
targetVertex, "Could not find target vertex: " + vertexRelationship.getTargetId());
String label = vertexRelationship.getLabel();
checkNotNull(label, "label is required");
Edge edge =
singleOrDefault(
sourceTermMentionsWithGraphVertex
.getVertex()
.getEdges(targetVertex, Direction.OUT, label, getAuthorizations()),
null);
if (edge == null) {
LOGGER.debug(
"adding edge %s -> %s (%s)",
sourceTermMentionsWithGraphVertex.getVertex().getId(), targetVertex.getId(), label);
graph.addEdge(
sourceTermMentionsWithGraphVertex.getVertex(),
targetVertex,
label,
vertexRelationship.getVisibility(),
getAuthorizations());
}
}
}
private void saveRelationships(
List<TermRelationship> relationships,
List<TermMentionWithGraphVertex> termMentionsWithGraphVertices) {
for (TermRelationship relationship : relationships) {
TermMentionWithGraphVertex sourceTermMentionsWithGraphVertex =
findTermMentionWithGraphVertex(
termMentionsWithGraphVertices, relationship.getSourceTermMention());
checkNotNull(
sourceTermMentionsWithGraphVertex,
"source was not found for " + relationship.getSourceTermMention());
checkNotNull(
sourceTermMentionsWithGraphVertex.getVertex(),
"source vertex was not found for " + relationship.getSourceTermMention());
TermMentionWithGraphVertex destTermMentionsWithGraphVertex =
findTermMentionWithGraphVertex(
termMentionsWithGraphVertices, relationship.getDestTermMention());
checkNotNull(
destTermMentionsWithGraphVertex,
"dest was not found for " + relationship.getDestTermMention());
checkNotNull(
destTermMentionsWithGraphVertex.getVertex(),
"dest vertex was not found for " + relationship.getDestTermMention());
String label = relationship.getLabel();
// TODO: a better way to check if the same edge exists instead of looking it up every time?
Edge edge =
singleOrDefault(
sourceTermMentionsWithGraphVertex
.getVertex()
.getEdges(
destTermMentionsWithGraphVertex.getVertex(),
Direction.OUT,
label,
getAuthorizations()),
null);
if (edge == null) {
graph.addEdge(
sourceTermMentionsWithGraphVertex.getVertex(),
destTermMentionsWithGraphVertex.getVertex(),
label,
relationship.getVisibility(),
getAuthorizations());
}
}
graph.flush();
}
private TermMentionWithGraphVertex findTermMentionWithGraphVertex(
List<TermMentionWithGraphVertex> termMentionsWithGraphVertices, TermMention termMention) {
for (TermMentionWithGraphVertex termMentionsWithGraphVertex : termMentionsWithGraphVertices) {
if (termMentionsWithGraphVertex.getTermMention().getRowKey().getStartOffset()
== termMention.getStart()
&& termMentionsWithGraphVertex.getTermMention().getRowKey().getEndOffset()
== termMention.getEnd()
&& termMentionsWithGraphVertex.getVertex() != null) {
return termMentionsWithGraphVertex;
}
}
return null;
}
protected List<TermMentionWithGraphVertex> saveTermMentions(
Vertex artifactGraphVertex,
Iterable<TermMention> termMentions,
String workspaceId,
String visibilitySource) {
getAuditRepository()
.auditAnalyzedBy(
AuditAction.ANALYZED_BY,
artifactGraphVertex,
getClass().getSimpleName(),
getUser(),
artifactGraphVertex.getVisibility());
for (TermMentionFilter termMentionFilter : this.workerPrepareData.getTermMentionFilters()) {
try {
termMentions = termMentionFilter.apply(artifactGraphVertex, termMentions);
} catch (Exception ex) {
throw new LumifyException(
"Failed to run term mention filter: " + termMentionFilter.getClass().getName(), ex);
}
}
List<TermMentionWithGraphVertex> results = new ArrayList<TermMentionWithGraphVertex>();
for (TermMention termMention : termMentions) {
results.add(saveTermMention(artifactGraphVertex, termMention, workspaceId, visibilitySource));
}
return results;
}
private TermMentionWithGraphVertex saveTermMention(
Vertex artifactGraphVertex,
TermMention termMention,
String workspaceId,
String visibilitySource) {
LOGGER.debug(
"Saving term mention '%s':%s:%s (%d:%d)",
termMention.getSign(),
termMention.getOntologyClassUri(),
termMention.getPropertyKey(),
termMention.getStart(),
termMention.getEnd());
Vertex vertex = null;
if (visibilitySource == null) {
visibilitySource = termMention.getVisibility().getVisibilityString();
}
JSONObject visibilityJson = new JSONObject();
visibilityJson.put(VisibilityTranslator.JSON_SOURCE, visibilitySource);
Visibility visibility = visibilityTranslator.toVisibility(visibilityJson).getVisibility();
Visibility visibilityWithWorkspaceId;
JSONObject visibilityJsonWithWorkspaceId;
if (!workspaceId.equals("")) {
visibilityJsonWithWorkspaceId =
GraphUtil.updateVisibilitySourceAndAddWorkspaceId(null, visibilitySource, workspaceId);
visibilityWithWorkspaceId =
visibilityTranslator.toVisibility(visibilityJsonWithWorkspaceId).getVisibility();
} else {
visibilityWithWorkspaceId = visibility;
visibilityJsonWithWorkspaceId = visibilityJson;
}
TermMentionRowKey termMentionRowKey =
new TermMentionRowKey(
artifactGraphVertex.getId().toString(),
termMention.getPropertyKey(),
termMention.getStart(),
termMention.getEnd());
TermMentionModel termMentionModel = new TermMentionModel(termMentionRowKey);
termMentionModel.getMetadata().setSign(termMention.getSign(), visibility);
termMentionModel
.getMetadata()
.setOntologyClassUri(termMention.getOntologyClassUri(), visibility);
if (termMention.getProcess() != null && !termMention.getProcess().equals("")) {
termMentionModel.getMetadata().setAnalyticProcess(termMention.getProcess(), visibility);
}
Concept concept = ontologyRepository.getConceptByIRI(termMention.getOntologyClassUri());
if (concept == null) {
LOGGER.error("Could not find ontology graph vertex '%s'", termMention.getOntologyClassUri());
return null;
}
termMentionModel.getMetadata().setConceptGraphVertexId(concept.getTitle(), visibility);
if (termMention.isResolved()) {
String title = termMention.getSign();
ElementMutation<Vertex> vertexElementMutation;
if (termMention.getUseExisting()) {
graph.flush(); // make sure the previous term mentions have made it into the graph
if (termMention.getId() != null) {
vertex = graph.getVertex(termMention.getId(), getAuthorizations());
} else {
vertex =
singleOrDefault(
graph
.query(getAuthorizations())
.has(LumifyProperties.TITLE.getPropertyName(), title)
.has(LumifyProperties.CONCEPT_TYPE.getPropertyName(), concept.getTitle())
.vertices(),
null);
}
}
Map<String, Object> metadata = new HashMap<String, Object>();
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setMetadata(
metadata, visibilityJsonWithWorkspaceId);
if (vertex == null) {
if (termMention.getId() != null) {
vertexElementMutation =
graph.prepareVertex(termMention.getId(), visibilityWithWorkspaceId);
} else {
vertexElementMutation = graph.prepareVertex(visibilityWithWorkspaceId);
}
LumifyProperties.TITLE.setProperty(
vertexElementMutation, title, metadata, visibilityWithWorkspaceId);
LumifyProperties.CONCEPT_TYPE.setProperty(
vertexElementMutation, concept.getTitle(), metadata, visibilityWithWorkspaceId);
} else {
vertexElementMutation = vertex.prepareMutation();
}
for (TermMention.TermMentionProperty termMentionProperty : termMention.getNewProperties()) {
vertexElementMutation.addPropertyValue(
termMentionProperty.getKey(),
termMentionProperty.getName(),
termMentionProperty.getValue(),
metadata,
visibilityWithWorkspaceId);
}
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
vertexElementMutation,
visibilityJsonWithWorkspaceId,
metadata,
visibilityWithWorkspaceId);
vertexElementMutation.addPropertyValue(
graph.getIdGenerator().nextId(),
LumifyProperties.ROW_KEY.getPropertyName(),
termMentionRowKey.toString(),
metadata,
visibilityWithWorkspaceId);
if (!(vertexElementMutation instanceof ExistingElementMutation)) {
vertex = vertexElementMutation.save(getAuthorizations());
auditRepository.auditVertexElementMutation(
AuditAction.UPDATE,
vertexElementMutation,
vertex,
termMention.getProcess(),
getUser(),
visibilityWithWorkspaceId);
} else {
auditRepository.auditVertexElementMutation(
AuditAction.UPDATE,
vertexElementMutation,
vertex,
termMention.getProcess(),
getUser(),
visibilityWithWorkspaceId);
vertex = vertexElementMutation.save(getAuthorizations());
}
// TODO: a better way to check if the same edge exists instead of looking it up every time?
Edge edge =
singleOrDefault(
artifactGraphVertex.getEdges(
vertex, Direction.OUT, artifactHasEntityIri, getAuthorizations()),
null);
if (edge == null) {
edge =
graph.addEdge(
artifactGraphVertex, vertex, artifactHasEntityIri, visibility, getAuthorizations());
LumifyVisibilityProperties.VISIBILITY_JSON_PROPERTY.setProperty(
edge,
visibilityJsonWithWorkspaceId,
metadata,
visibilityWithWorkspaceId,
getAuthorizations());
auditRepository.auditRelationship(
AuditAction.CREATE,
artifactGraphVertex,
vertex,
edge,
termMention.getProcess(),
"",
getUser(),
visibilityWithWorkspaceId);
}
graph.flush();
if (workspaceId != null && !workspaceId.equals("")) {
Workspace workspace = workspaceRepository.findById(workspaceId, getUser());
workspaceRepository.updateEntityOnWorkspace(
workspace, vertex.getId(), null, null, null, getUser());
}
termMentionModel
.getMetadata()
.setVertexId(vertex.getId().toString(), visibility)
.setEdgeId(edge.getId().toString(), visibility);
}
getTermMentionRepository().save(termMentionModel, FlushFlag.NO_FLUSH);
return new TermMentionWithGraphVertex(termMentionModel, vertex);
}
public static class TermMentionWithGraphVertex {
private final TermMentionModel termMention;
private final Vertex vertex;
public TermMentionWithGraphVertex(TermMentionModel termMention, Vertex vertex) {
this.termMention = termMention;
this.vertex = vertex;
}
public TermMentionModel getTermMention() {
return termMention;
}
public Vertex getVertex() {
return vertex;
}
}
}
public class ImportMRMapper extends ElementMapper<LongWritable, Text, Text, Mutation> {
private static final LumifyLogger LOGGER = LumifyLoggerFactory.getLogger(ImportMRMapper.class);
public static final String CONFIG_SOURCE_FILE_NAME = "sourceFileName";
private AccumuloGraph graph;
private Visibility visibility;
private Authorizations authorizations;
private SystemUser user;
private UserRepository userRepository;
private SecureGraphAuditRepository auditRepository;
private String sourceFileName;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
Map configurationMap = SecureGraphMRUtils.toMap(context.getConfiguration());
this.graph =
(AccumuloGraph)
new GraphFactory().createGraph(MapUtils.getAllWithPrefix(configurationMap, "graph"));
this.visibility = new Visibility("");
this.authorizations = new AccumuloAuthorizations();
this.user = new SystemUser(null);
VersionService versionService = new VersionService();
io.lumify.core.config.Configuration configuration =
new HashMapConfigurationLoader(configurationMap).createConfiguration();
this.auditRepository =
new SecureGraphAuditRepository(null, versionService, configuration, null, userRepository);
this.sourceFileName = context.getConfiguration().get(CONFIG_SOURCE_FILE_NAME);
}
@Override
protected void map(LongWritable filePosition, Text line, Context context)
throws IOException, InterruptedException {
try {
safeMap(filePosition, line, context);
} catch (Exception ex) {
LOGGER.error("failed mapping " + filePosition, ex);
}
}
private void safeMap(LongWritable filePosition, Text line, Context context)
throws IOException, InterruptedException {
String lineString = line.toString();
int colonOffet = lineString.indexOf(':');
if (colonOffet < 1) {
return;
}
long userId = Long.parseLong(lineString.substring(0, colonOffet));
context.setStatus("User: "******"Friendster MR", "", user, visibility);
context.write(key, AccumuloSession.createMutationFromRow(audit));
String friends = lineString.substring(colonOffet + 1).trim();
if ("notfound".equals(friends) || "private".equals(friends)) {
// do nothing?
} else {
String[] friendsArray = friends.split(",");
for (String friend : friendsArray) {
friend = friend.trim();
if (friend.length() == 0) {
continue;
}
long friendId = Long.parseLong(friend);
Vertex friendVertex = createUserVertex(friendId);
addEdge(
ImportMR.getFriendEdgeId(userVertex, friendVertex),
userVertex,
friendVertex,
FriendsterOntology.EDGE_LABEL_FRIEND,
visibility,
authorizations);
context.getCounter(FriendsterImportCounters.FRIEND_EDGES_CREATED).increment(1);
}
}
context.getCounter(FriendsterImportCounters.USERS_PROCESSED).increment(1);
}
private Vertex createUserVertex(long userId) {
String userVertexId = ImportMR.getUserVertexId(userId);
VertexBuilder userVertexBuilder = prepareVertex(userVertexId, visibility);
LumifyProperties.CONCEPT_TYPE.setProperty(
userVertexBuilder, FriendsterOntology.CONCEPT_TYPE_USER, visibility);
Metadata titleMetadata = new Metadata();
LumifyProperties.TITLE.addPropertyValue(
userVertexBuilder,
ImportMR.MULTI_VALUE_KEY,
"Friendster User " + userId,
titleMetadata,
visibility);
LumifyProperties.SOURCE.addPropertyValue(
userVertexBuilder, ImportMR.MULTI_VALUE_KEY, ImportMR.FRIENDSTER_SOURCE, visibility);
return userVertexBuilder.save(authorizations);
}
@Override
protected void saveDataMutation(Context context, Text dataTableName, Mutation m)
throws IOException, InterruptedException {
context.write(ImportMR.getKey(dataTableName.toString(), m.getRow()), m);
}
@Override
protected void saveEdgeMutation(Context context, Text edgesTableName, Mutation m)
throws IOException, InterruptedException {
context.write(ImportMR.getKey(edgesTableName.toString(), m.getRow()), m);
}
@Override
protected void saveVertexMutation(Context context, Text verticesTableName, Mutation m)
throws IOException, InterruptedException {
context.write(ImportMR.getKey(verticesTableName.toString(), m.getRow()), m);
}
@Override
public IdGenerator getIdGenerator() {
return this.graph.getIdGenerator();
}
@Inject
public void setUserRepository(UserRepository userRepository) {
this.userRepository = userRepository;
}
}
public abstract class OntologyRepositoryBase implements OntologyRepository {
private static final LumifyLogger LOGGER =
LumifyLoggerFactory.getLogger(OntologyRepositoryBase.class);
public void defineOntology(Authorizations authorizations) {
Concept rootConcept = getOrCreateConcept(null, OntologyRepository.ROOT_CONCEPT_IRI, "root");
Concept entityConcept =
getOrCreateConcept(rootConcept, OntologyRepository.ENTITY_CONCEPT_IRI, "thing");
addEntityGlyphIcon(entityConcept);
importBaseOwlFile(authorizations);
}
private void importBaseOwlFile(Authorizations authorizations) {
importResourceOwl("base.owl", "http://lumify.io", authorizations);
importResourceOwl("user.owl", "http://lumify.io/user", authorizations);
importResourceOwl("workspace.owl", "http://lumify.io/workspace", authorizations);
}
private void importResourceOwl(String fileName, String iri, Authorizations authorizations) {
InputStream baseOwlFile = OntologyRepositoryBase.class.getResourceAsStream(fileName);
checkNotNull(
baseOwlFile,
"Could not load resource " + OntologyRepositoryBase.class.getResource(fileName));
try {
importFile(baseOwlFile, IRI.create(iri), null, authorizations);
} catch (Exception e) {
throw new LumifyException("Could not import ontology file", e);
} finally {
IOUtils.closeQuietly(baseOwlFile);
}
}
private void addEntityGlyphIcon(Concept entityConcept) {
InputStream entityGlyphIconInputStream =
OntologyRepositoryBase.class.getResourceAsStream("entity.png");
checkNotNull(
entityGlyphIconInputStream,
"Could not load resource " + OntologyRepositoryBase.class.getResource("entity.png"));
try {
ByteArrayOutputStream imgOut = new ByteArrayOutputStream();
IOUtils.copy(entityGlyphIconInputStream, imgOut);
byte[] rawImg = imgOut.toByteArray();
addEntityGlyphIconToEntityConcept(entityConcept, rawImg);
} catch (IOException e) {
throw new LumifyException("invalid stream for glyph icon");
}
}
protected abstract void addEntityGlyphIconToEntityConcept(Concept entityConcept, byte[] rawImg);
public boolean isOntologyDefined() {
try {
Concept concept = getConceptByIRI(OntologyRepository.ROOT_CONCEPT_IRI);
return concept != null; // todo should check for more
} catch (Exception e) {
if (e.getMessage() != null
&& e.getMessage().contains(OntologyLumifyProperties.ONTOLOGY_TITLE.getPropertyName())) {
return false;
}
throw new RuntimeException(e);
}
}
@Override
public void importFile(File inFile, IRI documentIRI, Authorizations authorizations)
throws Exception {
if (!inFile.exists()) {
throw new LumifyException("File " + inFile + " does not exist");
}
File inDir = inFile.getParentFile();
FileInputStream inFileIn = new FileInputStream(inFile);
try {
importFile(inFileIn, documentIRI, inDir, authorizations);
} finally {
inFileIn.close();
}
}
private void importFile(
InputStream in, IRI documentIRI, File inDir, Authorizations authorizations) throws Exception {
byte[] inFileData = IOUtils.toByteArray(in);
Reader inFileReader = new InputStreamReader(new ByteArrayInputStream(inFileData));
OWLOntologyLoaderConfiguration config = new OWLOntologyLoaderConfiguration();
OWLOntologyManager m = createOwlOntologyManager(config, documentIRI);
OWLOntologyDocumentSource documentSource = new ReaderDocumentSource(inFileReader, documentIRI);
OWLOntology o = m.loadOntologyFromOntologyDocument(documentSource, config);
storeOntologyFile(new ByteArrayInputStream(inFileData), documentIRI);
for (OWLClass ontologyClass : o.getClassesInSignature()) {
if (!o.isDeclared(ontologyClass, false)) {
continue;
}
importOntologyClass(o, ontologyClass, inDir, authorizations);
}
for (OWLDataProperty dataTypeProperty : o.getDataPropertiesInSignature()) {
if (!o.isDeclared(dataTypeProperty, false)) {
continue;
}
importDataProperty(o, dataTypeProperty);
}
for (OWLObjectProperty objectProperty : o.getObjectPropertiesInSignature()) {
if (!o.isDeclared(objectProperty, false)) {
continue;
}
importObjectProperty(o, objectProperty);
}
for (OWLObjectProperty objectProperty : o.getObjectPropertiesInSignature()) {
if (!o.isDeclared(objectProperty, false)) {
continue;
}
importInverseOf(o, objectProperty);
}
}
public OWLOntologyManager createOwlOntologyManager(
OWLOntologyLoaderConfiguration config, IRI excludeDocumentIRI) throws Exception {
OWLOntologyManager m = OWLManager.createOWLOntologyManager();
config.setMissingImportHandlingStrategy(MissingImportHandlingStrategy.SILENT);
loadOntologyFiles(m, config, excludeDocumentIRI);
return m;
}
protected abstract void storeOntologyFile(InputStream inputStream, IRI documentIRI);
public void exportOntology(OutputStream out, IRI documentIRI) throws Exception {
OWLOntologyManager m = OWLManager.createOWLOntologyManager();
OWLOntologyLoaderConfiguration config = new OWLOntologyLoaderConfiguration();
config.setMissingImportHandlingStrategy(MissingImportHandlingStrategy.SILENT);
List<OWLOntology> loadedOntologies = loadOntologyFiles(m, config, null);
OWLOntology o = findOntology(loadedOntologies, documentIRI);
if (o == null) {
throw new LumifyException("Could not find ontology with iri " + documentIRI);
}
Writer fileWriter = new OutputStreamWriter(out);
new RDFXMLRenderer(o, fileWriter).render();
}
protected abstract List<OWLOntology> loadOntologyFiles(
OWLOntologyManager m, OWLOntologyLoaderConfiguration config, IRI excludedIRI)
throws Exception;
private OWLOntology findOntology(List<OWLOntology> loadedOntologies, IRI documentIRI) {
for (OWLOntology o : loadedOntologies) {
if (documentIRI.equals(o.getOntologyID().getOntologyIRI())) {
return o;
}
}
return null;
}
protected Concept importOntologyClass(
OWLOntology o, OWLClass ontologyClass, File inDir, Authorizations authorizations)
throws IOException {
String uri = ontologyClass.getIRI().toString();
if ("http://www.w3.org/2002/07/owl#Thing".equals(uri)) {
return getEntityConcept();
}
String label = getLabel(o, ontologyClass);
checkNotNull(label, "label cannot be null or empty: " + uri);
LOGGER.info("Importing ontology class " + uri + " (label: " + label + ")");
Concept parent = getParentConcept(o, ontologyClass, inDir, authorizations);
Concept result = getOrCreateConcept(parent, uri, label);
String color = getColor(o, ontologyClass);
if (color != null) {
result.setProperty(
OntologyLumifyProperties.COLOR.getPropertyName(),
color,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
}
String displayType = getDisplayType(o, ontologyClass);
if (displayType != null) {
result.setProperty(
OntologyLumifyProperties.DISPLAY_TYPE.getPropertyName(),
displayType,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
}
String titleFormula = getTitleFormula(o, ontologyClass);
if (titleFormula != null) {
result.setProperty(
OntologyLumifyProperties.TITLE_FORMULA.getPropertyName(),
titleFormula,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
}
String subtitleFormula = getSubtitleFormula(o, ontologyClass);
if (subtitleFormula != null) {
result.setProperty(
OntologyLumifyProperties.SUBTITLE_FORMULA.getPropertyName(),
subtitleFormula,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
}
String timeFormula = getTimeFormula(o, ontologyClass);
if (timeFormula != null) {
result.setProperty(
OntologyLumifyProperties.TIME_FORMULA.getPropertyName(),
timeFormula,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
}
boolean userVisible = getUserVisible(o, ontologyClass);
result.setProperty(
OntologyLumifyProperties.USER_VISIBLE.getPropertyName(),
userVisible,
OntologyRepository.VISIBILITY.getVisibility(),
authorizations);
String glyphIconFileName = getGlyphIconFileName(o, ontologyClass);
setIconProperty(
result,
inDir,
glyphIconFileName,
LumifyProperties.GLYPH_ICON.getPropertyName(),
authorizations);
String mapGlyphIconFileName = getMapGlyphIconFileName(o, ontologyClass);
setIconProperty(
result,
inDir,
mapGlyphIconFileName,
LumifyProperties.MAP_GLYPH_ICON.getPropertyName(),
authorizations);
return result;
}
private void setIconProperty(
Concept concept,
File inDir,
String glyphIconFileName,
String propertyKey,
Authorizations authorizations)
throws IOException {
if (glyphIconFileName != null) {
File iconFile = new File(inDir, glyphIconFileName);
if (!iconFile.exists()) {
throw new RuntimeException("Could not find icon file: " + iconFile.toString());
}
InputStream iconFileIn = new FileInputStream(iconFile);
try {
StreamingPropertyValue value = new StreamingPropertyValue(iconFileIn, byte[].class);
value.searchIndex(false);
value.store(true);
concept.setProperty(
propertyKey, value, OntologyRepository.VISIBILITY.getVisibility(), authorizations);
} finally {
iconFileIn.close();
}
}
}
protected Concept getParentConcept(
OWLOntology o, OWLClass ontologyClass, File inDir, Authorizations authorizations)
throws IOException {
Set<OWLClassExpression> superClasses = ontologyClass.getSuperClasses(o);
if (superClasses.size() == 0) {
return getEntityConcept();
} else if (superClasses.size() == 1) {
OWLClassExpression superClassExpr = superClasses.iterator().next();
OWLClass superClass = superClassExpr.asOWLClass();
String superClassUri = superClass.getIRI().toString();
Concept parent = getConceptByIRI(superClassUri);
if (parent != null) {
return parent;
}
parent = importOntologyClass(o, superClass, inDir, authorizations);
if (parent == null) {
throw new LumifyException("Could not find or create parent: " + superClass);
}
return parent;
} else {
throw new LumifyException(
"Unhandled multiple super classes. Found " + superClasses.size() + ", expected 0 or 1.");
}
}
protected void importDataProperty(OWLOntology o, OWLDataProperty dataTypeProperty) {
String propertyIRI = dataTypeProperty.getIRI().toString();
String propertyDisplayName = getLabel(o, dataTypeProperty);
PropertyType propertyType = getPropertyType(o, dataTypeProperty);
boolean userVisible = getUserVisible(o, dataTypeProperty);
boolean searchable = getSearchable(o, dataTypeProperty);
Boolean displayTime = getDisplayTime(o, dataTypeProperty);
Double boost = getBoost(o, dataTypeProperty);
if (propertyType == null) {
throw new LumifyException("Could not get property type on data property " + propertyIRI);
}
for (OWLClassExpression domainClassExpr : dataTypeProperty.getDomains(o)) {
OWLClass domainClass = domainClassExpr.asOWLClass();
String domainClassUri = domainClass.getIRI().toString();
Concept domainConcept = getConceptByIRI(domainClassUri);
checkNotNull(domainConcept, "Could not find class with uri: " + domainClassUri);
LOGGER.info("Adding data property " + propertyIRI + " to class " + domainConcept.getTitle());
ArrayList<PossibleValueType> possibleValues = getPossibleValues(o, dataTypeProperty);
Collection<TextIndexHint> textIndexHints = getTextIndexHints(o, dataTypeProperty);
addPropertyTo(
domainConcept,
propertyIRI,
propertyDisplayName,
propertyType,
possibleValues,
textIndexHints,
userVisible,
searchable,
displayTime,
boost);
}
}
protected abstract OntologyProperty addPropertyTo(
Concept concept,
String propertyIRI,
String displayName,
PropertyType dataType,
ArrayList<PossibleValueType> possibleValues,
Collection<TextIndexHint> textIndexHints,
boolean userVisible,
boolean searchable,
Boolean displayTime,
Double boost);
protected void importObjectProperty(OWLOntology o, OWLObjectProperty objectProperty) {
String iri = objectProperty.getIRI().toString();
String label = getLabel(o, objectProperty);
checkNotNull(label, "label cannot be null or empty for " + iri);
LOGGER.info("Importing ontology object property " + iri + " (label: " + label + ")");
for (Concept domain : getDomainsConcepts(o, objectProperty)) {
for (Concept range : getRangesConcepts(o, objectProperty)) {
getOrCreateRelationshipType(domain, range, iri, label);
}
}
}
protected void importInverseOf(OWLOntology o, OWLObjectProperty objectProperty) {
String iri = objectProperty.getIRI().toString();
Relationship fromRelationship = null;
for (OWLObjectPropertyExpression inverseOf : objectProperty.getInverses(o)) {
if (inverseOf instanceof OWLObjectProperty) {
if (fromRelationship == null) {
fromRelationship = getRelationshipByIRI(iri);
}
OWLObjectProperty inverseOfOWLObjectProperty = (OWLObjectProperty) inverseOf;
String inverseOfIri = inverseOfOWLObjectProperty.getIRI().toString();
Relationship inverseOfRelationship = getRelationshipByIRI(inverseOfIri);
getOrCreateInverseOfRelationship(fromRelationship, inverseOfRelationship);
}
}
}
protected abstract void getOrCreateInverseOfRelationship(
Relationship fromRelationship, Relationship inverseOfRelationship);
private Iterable<Concept> getRangesConcepts(OWLOntology o, OWLObjectProperty objectProperty) {
String uri = objectProperty.getIRI().toString();
if (objectProperty.getRanges(o).size() == 0) {
throw new LumifyException("Invalid number of range properties on " + uri);
}
List<Concept> ranges = new ArrayList<Concept>();
for (OWLClassExpression rangeClassExpr : objectProperty.getRanges(o)) {
OWLClass rangeClass = rangeClassExpr.asOWLClass();
String rangeClassUri = rangeClass.getIRI().toString();
Concept ontologyClass = getConceptByIRI(rangeClassUri);
checkNotNull(ontologyClass, "Could not find class with uri: " + rangeClassUri);
ranges.add(ontologyClass);
}
return ranges;
}
private Iterable<Concept> getDomainsConcepts(OWLOntology o, OWLObjectProperty objectProperty) {
String uri = objectProperty.getIRI().toString();
if (objectProperty.getDomains(o).size() == 0) {
throw new LumifyException("Invalid number of domain properties on " + uri);
}
List<Concept> domains = new ArrayList<Concept>();
for (OWLClassExpression rangeClassExpr : objectProperty.getDomains(o)) {
OWLClass rangeClass = rangeClassExpr.asOWLClass();
String rangeClassUri = rangeClass.getIRI().toString();
Concept ontologyClass = getConceptByIRI(rangeClassUri);
checkNotNull(ontologyClass, "Could not find class with uri: " + rangeClassUri);
domains.add(ontologyClass);
}
return domains;
}
protected PropertyType getPropertyType(OWLOntology o, OWLDataProperty dataTypeProperty) {
Set<OWLDataRange> ranges = dataTypeProperty.getRanges(o);
if (ranges.size() == 0) {
return null;
}
if (ranges.size() > 1) {
throw new LumifyException(
"Unexpected number of ranges on data property " + dataTypeProperty.getIRI().toString());
}
for (OWLDataRange range : ranges) {
if (range instanceof OWLDatatype) {
OWLDatatype datatype = (OWLDatatype) range;
return getPropertyType(datatype.getIRI().toString());
}
}
throw new LumifyException(
"Could not find range on data property " + dataTypeProperty.getIRI().toString());
}
private PropertyType getPropertyType(String iri) {
if ("http://www.w3.org/2001/XMLSchema#string".equals(iri)) {
return PropertyType.STRING;
}
if ("http://www.w3.org/2001/XMLSchema#dateTime".equals(iri)) {
return PropertyType.DATE;
}
if ("http://www.w3.org/2001/XMLSchema#int".equals(iri)) {
return PropertyType.DOUBLE;
}
if ("http://www.w3.org/2001/XMLSchema#double".equals(iri)) {
return PropertyType.DOUBLE;
}
if ("http://lumify.io#geolocation".equals(iri)) {
return PropertyType.GEO_LOCATION;
}
if ("http://lumify.io#currency".equals(iri)) {
return PropertyType.CURRENCY;
}
if ("http://lumify.io#image".equals(iri)) {
return PropertyType.IMAGE;
}
if ("http://www.w3.org/2001/XMLSchema#hexBinary".equals(iri)) {
return PropertyType.BINARY;
}
if ("http://www.w3.org/2001/XMLSchema#boolean".equals(iri)) {
return PropertyType.BOOLEAN;
}
if ("http://www.w3.org/2001/XMLSchema#integer".equals(iri)) {
return PropertyType.INTEGER;
}
throw new LumifyException("Unhandled property type " + iri);
}
public static String getLabel(OWLOntology o, OWLEntity owlEntity) {
for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
if (annotation.getProperty().isLabel()) {
OWLLiteral value = (OWLLiteral) annotation.getValue();
return value.getLiteral();
}
}
return owlEntity.getIRI().toString();
}
protected String getColor(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(o, owlEntity, OntologyLumifyProperties.COLOR.getPropertyName());
}
protected String getDisplayType(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.DISPLAY_TYPE.getPropertyName());
}
protected String getTitleFormula(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.TITLE_FORMULA.getPropertyName());
}
protected String getSubtitleFormula(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.SUBTITLE_FORMULA.getPropertyName());
}
protected String getTimeFormula(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.TIME_FORMULA.getPropertyName());
}
protected Boolean getDisplayTime(OWLOntology o, OWLEntity owlEntity) {
String val =
getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.DISPLAY_TIME.getPropertyName());
if (val == null) {
return null;
}
return Boolean.parseBoolean(val);
}
protected Double getBoost(OWLOntology o, OWLEntity owlEntity) {
String val =
getAnnotationValueByUri(o, owlEntity, OntologyLumifyProperties.BOOST.getPropertyName());
if (val == null) {
return null;
}
return Double.parseDouble(val);
}
protected boolean getUserVisible(OWLOntology o, OWLEntity owlEntity) {
String val =
getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.USER_VISIBLE.getPropertyName());
return val == null || Boolean.parseBoolean(val);
}
protected boolean getSearchable(OWLOntology o, OWLEntity owlEntity) {
String val = getAnnotationValueByUri(o, owlEntity, "http://lumify.io#searchable");
return val == null || Boolean.parseBoolean(val);
}
protected String getGlyphIconFileName(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.GLYPH_ICON_FILE_NAME.getPropertyName());
}
protected String getMapGlyphIconFileName(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.MAP_GLYPH_ICON_FILE_NAME.getPropertyName());
}
protected ArrayList<PossibleValueType> getPossibleValues(OWLOntology o, OWLEntity owlEntity) {
return getAnnotationValuesByUri(
o, owlEntity, OntologyLumifyProperties.POSSIBLE_VALUES.getPropertyName());
}
protected Collection<TextIndexHint> getTextIndexHints(OWLOntology o, OWLDataProperty owlEntity) {
return TextIndexHint.parse(
getAnnotationValueByUri(
o, owlEntity, OntologyLumifyProperties.TEXT_INDEX_HINTS.getPropertyName()));
}
protected ArrayList<PossibleValueType> getAnnotationValuesByUri(
OWLOntology o, OWLEntity owlEntity, String uri) {
ArrayList<PossibleValueType> possibleValueTypes = new ArrayList<PossibleValueType>();
for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
if (annotation.getProperty().getIRI().toString().equals(uri)) {
OWLLiteral value = (OWLLiteral) annotation.getValue();
String[] possibleValues = value.getLiteral().split(",");
for (String possibleValue : possibleValues) {
String[] parsedValue = possibleValue.split(":");
checkArgument(
parsedValue.length == 2, "Possible values must be in the format mapping:value");
possibleValueTypes.add(new PossibleValueType(parsedValue[0], parsedValue[1]));
}
}
}
return possibleValueTypes;
}
private String getAnnotationValueByUri(OWLOntology o, OWLEntity owlEntity, String uri) {
for (OWLAnnotation annotation : owlEntity.getAnnotations(o)) {
if (annotation.getProperty().getIRI().toString().equals(uri)) {
OWLLiteral value = (OWLLiteral) annotation.getValue();
return value.getLiteral();
}
}
return null;
}
@Override
public void writePackage(File file, IRI documentIRI, Authorizations authorizations)
throws Exception {
ZipFile zipped = new ZipFile(file);
if (zipped.isValidZipFile()) {
File tempDir = Files.createTempDir();
try {
LOGGER.info("Extracting: %s to %s", file.getAbsoluteFile(), tempDir.getAbsolutePath());
zipped.extractAll(tempDir.getAbsolutePath());
File owlFile = findOwlFile(tempDir);
importFile(owlFile, documentIRI, authorizations);
} finally {
FileUtils.deleteDirectory(tempDir);
}
} else {
importFile(file, documentIRI, authorizations);
}
}
protected File findOwlFile(File dir) {
File[] files = dir.listFiles();
if (files == null) {
return null;
}
for (File child : files) {
if (child.isDirectory()) {
File found = findOwlFile(child);
if (found != null) {
return found;
}
} else if (child.getName().toLowerCase().endsWith(".owl")) {
return child;
}
}
return null;
}
@Override
public void resolvePropertyIds(JSONArray filterJson) throws JSONException {
for (int i = 0; i < filterJson.length(); i++) {
JSONObject filter = filterJson.getJSONObject(i);
if (filter.has("propertyId") && !filter.has("propertyName")) {
String propertyVertexId = filter.getString("propertyId");
OntologyProperty property = getProperty(propertyVertexId);
if (property == null) {
throw new RuntimeException("Could not find property with id: " + propertyVertexId);
}
filter.put("propertyName", property.getTitle());
filter.put("propertyDataType", property.getDataType());
}
}
}
}
