private void getSheetContent(String jsonString, HSSFSheet sheet) {
try {
MapType mt =
TypeFactory.defaultInstance().constructMapType(HashMap.class, String.class, String.class);
CollectionType ct =
TypeFactory.defaultInstance().constructCollectionType(ArrayList.class, mt);
List<Map<String, String>> prms = JsonParser.json2Object(ct, jsonString);
for (int i = 0; i < prms.size(); i++) {
HSSFRow rowi = sheet.createRow(i);
Set<Entry<String, String>> set = prms.get(i).entrySet();
for (Entry<String, String> en : set) {
if (!"id".equals(en.getKey()) && !"displayOrder".equals(en.getKey())) {
if (en.getValue() != null) {
String value = en.getValue();
if ("name".equals(en.getKey())) {
HSSFCell celli0 = rowi.createCell(0);
celli0.setCellValue(value);
continue;
}
if ("dbColumnName".equals(en.getKey())) {
HSSFCell celli1 = rowi.createCell(1);
celli1.setCellValue(value);
continue;
}
if ("alias".equals(en.getKey())) {
HSSFCell celli2 = rowi.createCell(2);
celli2.setCellValue(value);
continue;
}
if ("dataType".equals(en.getKey())) {
HSSFCell celli3 = rowi.createCell(3);
celli3.setCellValue(value);
continue;
}
if ("defaultValue".equals(en.getKey())) {
HSSFCell celli4 = rowi.createCell(4);
celli4.setCellValue(value);
continue;
}
if ("maxLength".equals(en.getKey())) {
HSSFCell celli5 = rowi.createCell(5);
celli5.setCellValue(value);
continue;
}
}
}
}
HSSFCell celli6 = rowi.createCell(6);
celli6.setCellValue(i);
}
} catch (Exception e) {
}
}
public ListenableFuture<Map<JobId, JobStatus>> jobStatuses(final Set<JobId> jobs) {
final ConvertResponseToPojo<Map<JobId, JobStatus>> converter =
ConvertResponseToPojo.create(
TypeFactory.defaultInstance().constructMapType(Map.class, JobId.class, JobStatus.class),
ImmutableSet.of(HTTP_OK));
return transform(request(uri("/jobs/statuses"), "POST", jobs), converter);
}
/**
* Add the route in the database if it's not already, or update its state.
*
* @param routes being added to the app
*/
@Override
public void onRoutesAdd(Collection<Route> routes) {
for (Route routeCamel : routes) {
// adding a performance counter on the route
if (routeCamel instanceof EventDrivenConsumerRoute) {
EventDrivenConsumerRoute edcr = (EventDrivenConsumerRoute) routeCamel;
Processor processor = edcr.getProcessor();
if (processor instanceof InstrumentationProcessor) {
InstrumentationProcessor ip = (InstrumentationProcessor) processor;
ConsolePerformanceCounter counter =
new ConsolePerformanceCounter(routeCamel.getId(), repository);
ip.setCounter(counter);
log.debug("Adding a counter" + counter.toString() + " to " + routeCamel.getId());
}
}
// saving route in database
log.debug("Route added " + routeCamel.getId());
com.ninja_squad.console.Route route = repository.findRoute(routeCamel.getId());
if (route == null) {
route =
new com.ninja_squad.console.Route(routeCamel.getId())
.state(State.Started)
.uri(routeCamel.getEndpoint().getEndpointUri());
ObjectMapper mapper = new ObjectMapper();
AnnotationIntrospector introspector =
new JaxbAnnotationIntrospector(TypeFactory.defaultInstance());
// make serializer use JAXB annotations (only)
mapper.setAnnotationIntrospector(introspector);
String definition = null;
RouteDefinition routeDefinition = routeCamel.getRouteContext().getRoute();
try {
definition = mapper.writeValueAsString(routeDefinition);
} catch (IOException e) {
log.error("Error while marshalling route definition", e);
}
route.setDefinition(definition);
for (ProcessorDefinition<?> stepDefinition : routeDefinition.getOutputs()) {
if (stepDefinition.getId() == null) {
stepDefinition.setId(stepDefinition.getClass().getSimpleName());
}
route.getSteps().put(stepDefinition.getId(), stepDefinition.getLabel());
}
repository.save(route);
}
// saving state in database
RouteState routeState = repository.lastRouteState(routeCamel.getId());
if (routeState == null || routeState.getState().equals(State.Stopped)) {
routeState = new RouteState();
routeState.setRouteId(routeCamel.getId());
routeState.setState(State.Started);
routeState.setTimestamp(DateTime.now().getMillis());
repository.save(routeState);
}
}
}
public ListenableFuture<Map<String, HostStatus>> hostStatuses(
final List<String> hosts, final Map<String, String> queryParams) {
final ConvertResponseToPojo<Map<String, HostStatus>> converter =
ConvertResponseToPojo.create(
TypeFactory.defaultInstance()
.constructMapType(Map.class, String.class, HostStatus.class),
ImmutableSet.of(HTTP_OK));
return transform(request(uri("/hosts/statuses", queryParams), "POST", hosts), converter);
}
/**
* 把JSON转成list中是MAP《String，clazz》
*
* @param json
* @param clazz
* @return
*/
@SuppressWarnings({"rawtypes"})
public Object getJsonToListByMap(String json, Class clazz) {
Object object = null;
if (StringUtils.isNotBlank(json)) {
try {
object =
getMapper()
.readValue(
json,
TypeFactory.defaultInstance()
.constructArrayType(
TypeFactory.defaultInstance()
.constructMapType(HashMap.class, String.class, clazz)));
} catch (Exception e) {
e.printStackTrace();
}
}
return object;
}
public static XmlMapper newXmlMapper() {
XmlMapper xmlMapper = new XmlMapper();
final AnnotationIntrospector primary = new JacksonAnnotationIntrospector();
final AnnotationIntrospector secondary =
new JaxbAnnotationIntrospector(TypeFactory.defaultInstance());
final AnnotationIntrospector pair = new AnnotationIntrospectorPair(primary, secondary);
xmlMapper.setAnnotationIntrospector(pair);
xmlMapper.registerModule(new JodaModule());
xmlMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
return xmlMapper;
}
/**
* 把JSON转成Map<> value是LIST类型
*
* @param json
* @param keyclazz
* @param valueclazz LIST中的CLASS
* @return
*/
@SuppressWarnings("rawtypes")
public Object getJsonToMapByList(String json, Class keyclazz, Class valueclazz) {
Object object = null;
if (StringUtils.isNotBlank(json)) {
try {
object =
getMapper()
.readValue(
json,
TypeFactory.defaultInstance()
.constructMapType(
HashMap.class,
TypeFactory.defaultInstance().uncheckedSimpleType(keyclazz),
TypeFactory.defaultInstance()
.constructCollectionType(ArrayList.class, valueclazz)));
} catch (Exception e) {
e.printStackTrace();
}
}
return object;
}
/**
* Additional simple tests to ensure we will retain basic namespace information now that it can be
* included
*
* @since 2.1
*/
public void testNamespaces() throws Exception {
JaxbAnnotationIntrospector ai = new JaxbAnnotationIntrospector(TypeFactory.defaultInstance());
AnnotatedClass ac = AnnotatedClass.construct(NamespaceBean.class, ai, null);
AnnotatedField af = _findField(ac, "string");
assertNotNull(af);
PropertyName pn = ai.findNameForDeserialization(af);
assertNotNull(pn);
// JAXB seems to assert field name instead of giving "use default"...
assertEquals("", pn.getSimpleName());
assertEquals("urn:method", pn.getNamespace());
}
@Override
public Object fromBody(TypedInput body, final Type type) throws ConversionException {
try {
final JavaType javaType = TypeFactory.defaultInstance().constructType(type);
return objectMapper.readValue(body.in(), javaType);
} catch (final JsonParseException e) {
throw new ConversionException(e);
} catch (final JsonMappingException e) {
throw new ConversionException(e);
} catch (final IOException e) {
throw new ConversionException(e);
}
}
public byte[] getJsonTobyteArray(String json) {
byte[] object = null;
if (StringUtils.isNotBlank(json)) {
try {
object =
getMapper()
.readValue(json, TypeFactory.defaultInstance().constructArrayType(byte.class));
} catch (Exception e) {
e.printStackTrace();
}
}
return object;
}
/**
* Invokes the given method on the {@code handler} passing the given params (after converting them
* to beans\objects) to it.
*
* @param m the method to invoke
* @param params the params to pass to the method
* @return the return value (or null if no return)
* @throws IOException on error
* @throws IllegalAccessException on error
* @throws InvocationTargetException on error
*/
protected JsonNode invoke(Method m, List<JsonNode> params)
throws IOException, IllegalAccessException, InvocationTargetException {
// debug log
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log(Level.FINE, "Invoking method: " + m.getName());
}
// convert the parameters
Object[] convertedParams = new Object[params.size()];
Type[] parameterTypes = m.getGenericParameterTypes();
for (int i = 0; i < parameterTypes.length; i++) {
JsonParser paramJsonParser = mapper.treeAsTokens(params.get(i));
JavaType paramJavaType = TypeFactory.defaultInstance().constructType(parameterTypes[i]);
convertedParams[i] = mapper.readValue(paramJsonParser, paramJavaType);
}
// invoke the method
Object result = m.invoke(handler, convertedParams);
Type genericReturnType = m.getGenericReturnType();
if (genericReturnType != null) {
if (Collection.class.isInstance(result) && genericReturnType instanceof ParameterizedType) {
try {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log(Level.FINE, "attempting custom collection serialization");
}
TypeFactory typeFactory = mapper.getTypeFactory();
Type actualTypeInCollection =
((ParameterizedType) genericReturnType).getActualTypeArguments()[0];
if (actualTypeInCollection
instanceof TypeVariable) { // collection actually has a generic return type
actualTypeInCollection = ((TypeVariable) actualTypeInCollection).getBounds()[0];
}
JavaType rootType =
typeFactory.constructCollectionType(
Collection.class, typeFactory.constructType(actualTypeInCollection));
return valueToTree(mapper.writerWithType(rootType), result);
} catch (Exception e) {
LOGGER.log(
Level.WARNING,
"could not do custom collection serialization falling back to default",
e);
}
}
return mapper.valueToTree(result);
} else {
return null;
}
// return (genericReturnType!=null) ? mapper.valueToTree(result) : null;
}
/**
* 把JSON转成Object
*
* @param json
* @param keyclazz
* @param valueclazz
* @return
*/
@SuppressWarnings({"rawtypes"})
public Object getJsonToObject(String json, Class objclazz, Class... pclazz) {
Object object = null;
if (StringUtils.isNotBlank(json)) {
try {
object =
getMapper()
.readValue(
json, TypeFactory.defaultInstance().constructParametricType(objclazz, pclazz));
} catch (Exception e) {
}
}
return object;
}
private static ObjectMapper createMapper() {
ObjectMapper mapper = new ObjectMapper();
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
mapper.setDateFormat(dateFormat);
mapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(TypeFactory.defaultInstance()));
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true);
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
mapper.configure(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME, true);
addQueryMapper(mapper);
return mapper;
}
/**
* Parse Json File and create list of {@link WidgetGroup}.
*
* @param path
*/
public void read(String path) {
ObjectMapper mapper = new ObjectMapper();
try {
// JSON from file to Object
TypeFactory typeFactory = TypeFactory.defaultInstance();
String json = TextFileUtils.readFileFromJar(path);
widgetGroupList =
mapper.readValue(
json, typeFactory.constructCollectionType(ArrayList.class, DefaultWidgetGroup.class));
} catch (IOException ex) {
Logger.getLogger(JsonReader.class.getName()).log(Level.SEVERE, null, ex);
}
}
public void testSimpleKeySer() throws Exception {
ObjectMapper mapper = new ObjectMapper();
SimpleModule module = new SimpleModule("test", Version.unknownVersion());
module.addKeySerializer(String.class, new ContextualKeySerializer("prefix"));
mapper.registerModule(module);
Map<String, Object> input = new HashMap<String, Object>();
input.put("a", Integer.valueOf(3));
String json =
mapper
.writerWithType(
TypeFactory.defaultInstance()
.constructMapType(HashMap.class, String.class, Object.class))
.writeValueAsString(input);
assertEquals("{\"prefix:a\":3}", json);
}
public <T> List<T> readValueAsObjectList(String h, Class clazz) {
List<T> list = null;
try {
TypeFactory t = TypeFactory.defaultInstance();
list =
Utils.instance
.getObjectMapper()
.readValue(h, t.constructCollectionType(ArrayList.class, clazz));
return list;
} catch (JsonProcessingException e) {
e.printStackTrace();
return list;
} catch (IOException e) {
e.printStackTrace();
return list;
}
}
public void testRootNameAccess() throws Exception {
AnnotationIntrospector ai = new JaxbAnnotationIntrospector(TypeFactory.defaultInstance());
// If no @XmlRootElement, should get null (unless pkg has etc)
assertNull(ai.findRootName(AnnotatedClass.construct(SimpleBean.class, ai, null)));
// With @XmlRootElement, but no name, empty String
PropertyName rootName =
ai.findRootName(AnnotatedClass.construct(NamespaceBean.class, ai, null));
assertNotNull(rootName);
assertEquals("", rootName.getSimpleName());
assertEquals("urn:class", rootName.getNamespace());
// and otherwise explicit name
rootName = ai.findRootName(AnnotatedClass.construct(RootNameBean.class, ai, null));
assertNotNull(rootName);
assertEquals("test", rootName.getSimpleName());
assertNull(rootName.getNamespace());
}
/**
* 把JSON转成并发Map
*
* @param json
* @param keyclazz
* @param valueclazz
* @return
*/
@SuppressWarnings({"rawtypes"})
public Object getJsonToConcMap(String json, Class keyclazz, Class valueclazz) {
Object object = null;
if (StringUtils.isNotBlank(json)) {
try {
object =
getMapper()
.readValue(
json,
TypeFactory.defaultInstance()
.constructParametricType(ConcurrentHashMap.class, keyclazz, valueclazz));
} catch (Exception e) {
e.printStackTrace();
}
}
return object;
}
@Test
public void testTickerAdapter() throws IOException {
// Read in the JSON from the example resources
InputStream is = Btc38AdapterTest.class.getResourceAsStream("/example-ticker-data.json");
// Use Jackson to parse it
ObjectMapper mapper = new ObjectMapper();
TypeFactory t = TypeFactory.defaultInstance();
Map<String, Btc38TickerReturn> btc38AllTickers =
mapper.readValue(is, t.constructMapType(Map.class, String.class, Btc38TickerReturn.class));
Btc38TickerReturn tickerReturn = btc38AllTickers.get("doge");
Ticker ticker = Btc38Adapters.adaptTicker(tickerReturn.getTicker(), CurrencyPair.DOGE_BTC);
assertThat(ticker.getHigh()).isEqualTo(new BigDecimal("6.0e-7"));
assertThat(ticker.getLow()).isEqualTo(new BigDecimal("5.1e-7"));
assertThat(ticker.getLast()).isEqualTo(new BigDecimal("5.5e-7"));
assertThat(ticker.getVolume()).isEqualTo(new BigDecimal("9201658.258436"));
assertThat(ticker.getAsk()).isEqualTo(new BigDecimal("5.7e-7"));
assertThat(ticker.getBid()).isEqualTo(new BigDecimal("5.5e-7"));
}
public DataCenterTypeInfoResolver() {
super(
TypeFactory.defaultInstance().constructType(DataCenterInfo.class),
TypeFactory.defaultInstance());
}
private void add(Class<?> paramClass, JsonDeserializer<?> paramJsonDeserializer)
{
this._allDeserializers.put(TypeFactory.defaultInstance().constructType(paramClass), paramJsonDeserializer);
}
public void testSupport() {
assertTrue(MAPPER.canSerialize(String.class));
assertTrue(MAPPER.canDeserialize(TypeFactory.defaultInstance().constructType(String.class)));
}
/**
* Consumer to persist Baleen outputs into Postgres.
*
* <p>Assumes that the provided schema has the following tables. If these do not exist, then they
* are created when the consumer is first run. If they exist, but have the wrong definition, then
* the consumer will be unable to insert into them. A prefix, defaulting to 'baleen_', can be
* specified to avoid name conflicts.
*
* <ul>
* <li>docs
* <li>doc_metadata
* <li>entities
* <li>entity_geos
* </ul>
*
* Requires PostGIS 2 or later.
*
* @baleen.javadoc
*/
public class Postgres extends BaleenConsumer {
/**
* Connection to PostgreSQL
*
* @baleen.resource uk.gov.dstl.baleen.resources.SharedPostgresResource
*/
public static final String KEY_POSTGRES = "postgres";
@ExternalResource(key = KEY_POSTGRES)
private SharedPostgresResource postgresResource;
/**
* The Postgres schema containing the tables
*
* @baleen.config
*/
public static final String PARAM_SCHEMA = "schema";
@ConfigurationParameter(name = PARAM_SCHEMA, defaultValue = "")
private String schema;
/**
* The prefix to add to table names
*
* @baleen.config baleen_
*/
public static final String PARAM_PREFIX = "prefix";
@ConfigurationParameter(name = PARAM_PREFIX, defaultValue = "baleen_")
private String prefix;
/**
* Should a hash of the content be used to generate the ID? If false, then a hash of the Source
* URI is used instead.
*
* @baleen.config true
*/
public static final String PARAM_CONTENT_HASH_AS_ID = "contentHashAsId";
@ConfigurationParameter(name = PARAM_CONTENT_HASH_AS_ID, defaultValue = "true")
private boolean contentHashAsId = true;
private PreparedStatement insertDocStatement;
private PreparedStatement insertDocMetadataStatement;
private PreparedStatement insertEntityStatement;
private PreparedStatement insertEntityGeoStatement;
private static final ObjectMapper MAPPER = new ObjectMapper();
private static final MapLikeType MAP_LIKE_TYPE =
TypeFactory.defaultInstance().constructMapLikeType(Map.class, String.class, Object.class);
private static final String DOC_ROOT = "docs";
private static final String DOC_METADATA_ROOT = "doc_metadata";
private static final String ENTITY_ROOT = "entities";
private static final String ENTITY_GEO_ROOT = "entity_geos";
private static final String VARCHAR = "varchar";
private static final String CREATE_TABLE_PREFIX = "CREATE TABLE IF NOT EXISTS ";
private static final String INSERT_INTO_PREFIX = "INSERT INTO ";
@Override
public void doInitialize(UimaContext aContext) throws ResourceInitializationException {
checkVersion();
createTables();
try {
insertDocStatement =
postgresResource
.getConnection()
.prepareStatement(
INSERT_INTO_PREFIX
+ getTableName(DOC_ROOT)
+ " (externalId, type, source, content, language, processed, classification, caveats, releasability) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
Statement.RETURN_GENERATED_KEYS);
insertDocMetadataStatement =
postgresResource
.getConnection()
.prepareStatement(
INSERT_INTO_PREFIX
+ getTableName(DOC_METADATA_ROOT)
+ " (doc_key, name, value) VALUES (?, ?, ?)");
insertEntityStatement =
postgresResource
.getConnection()
.prepareStatement(
INSERT_INTO_PREFIX
+ getTableName(ENTITY_ROOT)
+ " (doc_key, externalId, type, value) VALUES (?, ?, ?, ?)",
Statement.RETURN_GENERATED_KEYS);
insertEntityGeoStatement =
postgresResource
.getConnection()
.prepareStatement(
INSERT_INTO_PREFIX
+ getTableName(ENTITY_GEO_ROOT)
+ " (entity_key, geo) VALUES (?, ST_GeomFromGeoJSON(?))");
} catch (SQLException e) {
throw new ResourceInitializationException(e);
}
}
/** Check that Postgres has at least version 2 of PostGIS installed */
private void checkVersion() throws ResourceInitializationException {
try (Statement s = postgresResource.getConnection().createStatement(); ) {
ResultSet rs = s.executeQuery("SELECT PostGIS_Lib_Version() AS version");
rs.next();
String version = rs.getString("version");
String[] versionParts = version.split("\\.");
Integer majorVersion = Integer.parseInt(versionParts[0]);
if (majorVersion < 2) {
throw new BaleenException("Unsupported PostGIS Version");
}
} catch (SQLException | NumberFormatException | NullPointerException e) {
getMonitor().error("Unable to retrieve PostGIS version");
throw new ResourceInitializationException(e);
} catch (BaleenException e) {
throw new ResourceInitializationException(e);
}
}
/** If the tables don't already exist, then create them. */
private void createTables() throws ResourceInitializationException {
try (Statement s = postgresResource.getConnection().createStatement(); ) {
s.execute(
CREATE_TABLE_PREFIX
+ getTableName(DOC_ROOT)
+ " (key serial primary key, externalId character varying, type character varying, source character varying, content character varying, language character varying, processed timestamp, classification character varying, caveats character varying[], releasability character varying[])");
s.execute(
CREATE_TABLE_PREFIX
+ getTableName(DOC_METADATA_ROOT)
+ " (key serial primary key, doc_key integer references "
+ getTableName(DOC_ROOT)
+ "(key), name character varying, value character varying)");
s.execute(
CREATE_TABLE_PREFIX
+ getTableName(ENTITY_ROOT)
+ " (key serial primary key, doc_key integer references "
+ getTableName(DOC_ROOT)
+ "(key), externalId character varying[], type character varying, value character varying[])");
s.execute(
CREATE_TABLE_PREFIX
+ getTableName(ENTITY_GEO_ROOT)
+ " (key serial primary key, entity_key integer references "
+ getTableName(ENTITY_ROOT)
+ "(key), geo geometry(Geometry, 4326))");
postgresResource.getConnection().setAutoCommit(false);
} catch (SQLException e) {
throw new ResourceInitializationException(e);
}
}
@Override
protected void doProcess(JCas jCas) throws AnalysisEngineProcessException {
Connection conn = postgresResource.getConnection();
try {
// Insert document and metadata into database
Integer docKey = executeDocInsert(jCas);
for (Metadata md : JCasUtil.select(jCas, Metadata.class)) {
executeDocMetadataInsert(docKey, md);
}
processEntities(jCas, docKey);
conn.commit();
} catch (SQLException | BaleenException e) {
getMonitor().error("Unable to insert document into Postgres database", e);
if (conn != null) {
try {
conn.rollback();
} catch (SQLException e2) {
getMonitor()
.error(
"Unable to rollback insertion - state of the database may have been left inconsistent",
e2);
}
}
}
}
private Integer executeDocInsert(JCas jCas) throws SQLException, BaleenException {
DocumentAnnotation da = getDocumentAnnotation(jCas);
String documentId = ConsumerUtils.getExternalId(da, contentHashAsId);
insertDocStatement.clearParameters();
insertDocStatement.setString(1, documentId);
insertDocStatement.setString(2, da.getDocType());
insertDocStatement.setString(3, da.getSourceUri());
insertDocStatement.setString(4, jCas.getDocumentText());
insertDocStatement.setString(5, jCas.getDocumentLanguage());
insertDocStatement.setTimestamp(6, new Timestamp(da.getTimestamp()));
insertDocStatement.setString(7, da.getDocumentClassification());
insertDocStatement.setArray(
8, createVarcharArray(postgresResource.getConnection(), da.getDocumentCaveats()));
insertDocStatement.setArray(
9, createVarcharArray(postgresResource.getConnection(), da.getDocumentReleasability()));
insertDocStatement.executeUpdate();
Integer docKey = getKey(insertDocStatement);
if (docKey == null) {
throw new BaleenException("No document key returned");
}
return docKey;
}
private void executeDocMetadataInsert(Integer docKey, Metadata md) throws SQLException {
insertDocMetadataStatement.clearParameters();
insertDocMetadataStatement.setInt(1, docKey);
insertDocMetadataStatement.setString(2, md.getKey());
insertDocMetadataStatement.setString(3, md.getValue());
insertDocMetadataStatement.executeUpdate();
}
private void processEntities(JCas jCas, Integer docKey) throws SQLException {
// Insert entities
Map<ReferenceTarget, List<Entity>> coreferenceEntities = new HashMap<>();
for (Entity ent : JCasUtil.select(jCas, Entity.class)) {
ReferenceTarget rt = ent.getReferent();
if (rt == null) {
rt = new ReferenceTarget(jCas);
}
List<Entity> entities = coreferenceEntities.getOrDefault(rt, new ArrayList<>());
entities.add(ent);
coreferenceEntities.put(rt, entities);
}
for (List<Entity> entities : coreferenceEntities.values()) {
processCoreferencedEntities(docKey, entities);
}
}
private void processCoreferencedEntities(Integer docKey, List<Entity> entities)
throws SQLException {
Set<String> values = new HashSet<>();
Set<String> externalIds = new HashSet<>();
Set<String> geoJsons = new HashSet<>();
Class<? extends Entity> type = null;
for (Entity e : entities) {
values.add(e.getValue());
externalIds.add(e.getExternalId());
if (e instanceof Location) {
Location l = (Location) e;
try {
geoJsons.add(addCrsToGeoJSON(l.getGeoJson()));
} catch (BaleenException ex) {
getMonitor().warn("Unable to add CRS to GeoJSON", ex);
}
}
type = getSuperclass(type, e.getClass());
}
Integer entityKey = executeEntityInsert(docKey, values, externalIds, type.getName());
if (entityKey != null) {
for (String geoJson : geoJsons) {
executeEntityGeoInsert(entityKey, geoJson);
}
}
}
private Integer executeEntityInsert(
Integer docKey, Collection<String> values, Collection<String> externalIds, String type)
throws SQLException {
insertEntityStatement.clearParameters();
insertEntityStatement.setInt(1, docKey);
insertEntityStatement.setArray(
2,
postgresResource
.getConnection()
.createArrayOf(VARCHAR, externalIds.toArray(new String[0])));
insertEntityStatement.setString(3, type);
insertEntityStatement.setArray(
4, postgresResource.getConnection().createArrayOf(VARCHAR, values.toArray(new String[0])));
insertEntityStatement.executeUpdate();
Integer entityKey = getKey(insertEntityStatement);
if (entityKey == null) {
getMonitor().error("No entity key returned - Geo insertion, if applicable, will be skipped");
}
return entityKey;
}
private void executeEntityGeoInsert(Integer entityKey, String geoJson) throws SQLException {
insertEntityGeoStatement.clearParameters();
insertEntityGeoStatement.setInt(1, entityKey);
insertEntityGeoStatement.setString(2, geoJson);
insertEntityGeoStatement.executeUpdate();
}
/** Add CRS (assumed to be EPSG:4326) to a GeoJSON string if it doesn't already exist */
public static String addCrsToGeoJSON(String geoJson) throws BaleenException {
if (Strings.isNullOrEmpty(geoJson)) {
return geoJson;
}
try {
Map<String, Object> geoJsonObj = MAPPER.readValue(geoJson, MAP_LIKE_TYPE);
if (geoJsonObj.get("crs") == null) {
Map<String, Object> crs = new HashMap<>();
crs.put("type", "name");
Map<String, Object> srid = new HashMap<>();
srid.put("name", "EPSG:4326");
crs.put("properties", srid);
geoJsonObj.put("crs", crs);
return MAPPER.writeValueAsString(geoJsonObj);
} else {
return geoJson;
}
} catch (Exception e) {
throw new BaleenException("Unable to parse GeoJSON", e);
}
}
/**
* From the two classes, return the one that is the superclass. If the two classes aren't in the
* same hierarchy, then c1 will be returned.
*/
public static Class<? extends Entity> getSuperclass(
Class<? extends Entity> c1, Class<? extends Entity> c2) {
if (c1 == null) return c2;
if (c2 == null) return c1;
if (c2.isAssignableFrom(c1)) {
return c2;
}
return c1;
}
/** Get the table name with the schema and prefix if one is set */
public String getTableName(String table) {
String ret = table;
if (!Strings.isNullOrEmpty(prefix)) {
ret = prefix + ret;
}
if (!Strings.isNullOrEmpty(schema)) {
ret = schema + "." + ret;
}
return ret;
}
private Array createVarcharArray(Connection conn, StringArray s) throws SQLException {
if (s == null) {
return conn.createArrayOf(VARCHAR, new String[] {});
} else {
return conn.createArrayOf(VARCHAR, s.toArray());
}
}
private Integer getKey(Statement s) throws SQLException {
ResultSet generatedKeys = s.getGeneratedKeys();
if (generatedKeys.next()) {
return generatedKeys.getInt(1);
} else {
return null;
}
}
}
public <T> T runOnce(Class<T> type, Operation<T> f) {
return runOnce(TypeFactory.defaultInstance().constructType(type), f);
}
/** Converts from an Entity or Relation into a Map representation, adding history if required. */
public class EntityRelationConverter {
private final boolean outputHistory;
private final DocumentHistory documentHistory;
private final Set<String> stopFeatures;
private final UimaMonitor monitor;
private final IEntityConverterFields fields;
private static final String FIELD_VALUE = "value";
private final ObjectMapper mapper = new ObjectMapper();
private final MapLikeType mapLikeType =
TypeFactory.defaultInstance().constructMapLikeType(Map.class, String.class, Object.class);
/**
* New instance.
*
* @param monitor the monitor to log to
* @param outputHistory true if should output history
* @param documentHistory the history of the document
* @param stopFeatures features which should be excluded from the serialisation
* @param fields fields to map properties to
*/
public EntityRelationConverter(
UimaMonitor monitor,
boolean outputHistory,
DocumentHistory documentHistory,
Set<String> stopFeatures,
IEntityConverterFields fields) {
this.monitor = monitor;
this.outputHistory = outputHistory;
this.documentHistory = documentHistory;
this.stopFeatures = stopFeatures;
this.fields = fields;
}
private UimaMonitor getMonitor() {
return monitor;
}
/**
* Convert from an entity to a map.
*
* @param entity the entity to convert
* @return a map containing the entity's fields (and history is required)
*/
public Map<String, Object> convertEntity(Entity entity) {
Map<String, Object> map = Maps.newHashMap();
convertFeatures(map, entity);
if (outputHistory && documentHistory != null) {
Collection<HistoryEvent> events = documentHistory.getHistory(entity.getInternalId());
convertHistory(map, events, entity.getInternalId());
}
putIfExists(map, fields.getExternalId(), entity.getExternalId());
return map;
}
/**
* Convert from a relation to a map.
*
* @param relation the relation to convert
* @return a map containing the relation's fields (and history is required)
*/
public Map<String, Object> convertRelation(Relation relation) {
Map<String, Object> map = Maps.newHashMap();
convertFeatures(map, relation);
if (outputHistory && documentHistory != null) {
Collection<HistoryEvent> events = documentHistory.getHistory(relation.getInternalId());
convertHistory(map, events, relation.getInternalId());
}
putIfExists(map, fields.getExternalId(), relation.getExternalId());
return map;
}
private void convertFeatures(Map<String, Object> map, Base base) {
for (Feature f : base.getType().getFeatures()) {
if (stopFeatures.contains(f.getName())) {
continue;
}
try {
convertFeature(map, base, f);
} catch (Exception e) {
getMonitor()
.warn(
"Couldn't output {} to map. Type '{}' isn't supported.",
f.getName(),
f.getRange().getShortName(),
e);
}
}
map.put("type", base.getType().getShortName());
if (map.get(FIELD_VALUE) == null || Strings.isNullOrEmpty(map.get(FIELD_VALUE).toString())) {
map.put(FIELD_VALUE, base.getCoveredText());
}
}
private void convertFeature(Map<String, Object> map, Base base, Feature f) {
if (f.getRange().isPrimitive()) {
if ("geoJson".equals(f.getShortName())) {
getMonitor().trace("Feature is GeoJSON - parsing to a database object");
putGeoJson(map, base.getFeatureValueAsString(f));
} else {
getMonitor().trace("Converting primitive feature to an object");
map.put(ConsumerUtils.toCamelCase(f.getShortName()), FeatureUtils.featureToObject(f, base));
}
} else if (f.getRange().isArray()
&& f.getRange().getComponentType() != null
&& f.getRange().getComponentType().isPrimitive()) {
getMonitor().trace("Converting primitive feature to an array");
map.put(ConsumerUtils.toCamelCase(f.getShortName()), FeatureUtils.featureToArray(f, base));
} else {
getMonitor()
.trace("Feature is not a primitive type - will try to treat the feature as an entity");
if (f.getRange().isArray()) {
getMonitor().trace("Feature is an array - attempting converstion to an array of entities");
FSArray fArr = (FSArray) base.getFeatureValue(f);
if (fArr != null) {
map.put(ConsumerUtils.toCamelCase(f.getShortName()), getEntityIds(fArr));
}
} else {
getMonitor().trace("Feature is singular - attempting conversion to a single entity");
FeatureStructure ent = base.getFeatureValue(f);
if (ent == null) {
// Ignore null entities
} else if (ent instanceof Entity) {
map.put(ConsumerUtils.toCamelCase(f.getShortName()), ((Entity) ent).getExternalId());
} else {
getMonitor().trace("Unable to persist feature {}", f.getShortName());
}
}
}
}
private void putGeoJson(Map<String, Object> map, String geojson) {
try {
if (!Strings.isNullOrEmpty(geojson)) {
putIfExists(map, fields.getGeoJSON(), mapper.readValue(geojson, mapLikeType));
}
} catch (IOException e) {
getMonitor().warn("Unable to persist geoJson", e);
}
}
private List<String> getEntityIds(FSArray entityArray) {
List<String> entities = new ArrayList<>();
for (int x = 0; x < entityArray.size(); x++) {
Entity ent = (Entity) entityArray.get(x);
entities.add(ent.getExternalId());
}
return entities;
}
private void convertHistory(
Map<String, Object> map, Collection<HistoryEvent> events, long entityInternalId) {
List<Object> list = new LinkedList<Object>();
saveEvents(list, events, entityInternalId);
putIfExists(map, fields.getHistory(), list);
}
private void saveEvents(
List<Object> list, Collection<HistoryEvent> events, long entityInternalId) {
for (HistoryEvent event : events) {
saveEvent(list, event, entityInternalId);
}
}
private void saveEvent(List<Object> list, HistoryEvent event, long entityInternalId) {
Map<String, Object> e = new LinkedHashMap<String, Object>();
if (event.getRecordable().getInternalId() != entityInternalId) {
// Only save the internal id as a reference to entities which aren't this one.
putIfExists(e, fields.getHistoryRecordable(), event.getRecordable().getInternalId());
}
putIfExists(e, fields.getHistoryAction(), event.getAction());
putIfExists(e, fields.getHistoryType(), event.getEventType());
putIfExists(e, fields.getHistoryParameters(), event.getParameters());
putIfExists(e, fields.getHistoryReferrer(), event.getReferrer());
putIfExists(e, fields.getHistoryTimestamp(), event.getTimestamp());
list.add(e);
if (HistoryEvents.MERGED_TYPE.equalsIgnoreCase(event.getEventType())
&& event.getParameters() != null
&& event.getParameters(HistoryEvents.PARAM_MERGED_ID).isPresent()) {
Optional<String> mergedId = event.getParameters(HistoryEvents.PARAM_MERGED_ID);
Integer id = Ints.tryParse(mergedId.get());
if (id != null) {
Collection<HistoryEvent> mergedEvents = documentHistory.getHistory(id);
if (mergedEvents != null) {
saveEvents(list, mergedEvents, entityInternalId);
} else {
getMonitor().warn("Null history for {}", id);
}
} else {
getMonitor().warn("No merge id for merge history of {}", event.getRecordable());
}
}
}
private void putIfExists(Map<String, Object> map, String key, Object value) {
if (!Strings.isNullOrEmpty(key)) {
map.put(key, value);
}
}
}
public class DefaultFailover implements Failover {
private static final Logger logger = LoggerFactory.getLogger(DefaultFailover.class);
private final TypeFactory factory = TypeFactory.defaultInstance();
private final ObjectMapper mapper = new ObjectMapper();
private final File file;
public DefaultFailover(File file) {
this.file = file;
}
public void onCompleteResponse(FullHttpRequest request, FullHttpResponse response) {
try {
ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
Session targetSession =
Session.newSession(
MessageFactory.createRequest(request), MessageFactory.createResponse(response));
writer.writeValue(this.file, prepareTargetSessions(targetSession));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private List<Session> prepareTargetSessions(Session targetSession) {
List<Session> sessions = restoreSessions(this.file);
Optional<Session> session = tryFind(sessions, isForRequest(targetSession.getRequest()));
if (session.isPresent()) {
session.get().setResponse(targetSession.getResponse());
} else {
sessions.add(targetSession);
}
return sessions;
}
private List<Session> restoreSessions(File file) {
try {
return mapper.readValue(file, factory.constructCollectionType(List.class, Session.class));
} catch (JsonMappingException jme) {
logger.error("exception found", jme);
return newArrayList();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public void failover(FullHttpRequest request, FullHttpResponse response) {
Response dumpedResponse = failoverResponse(request);
response.setProtocolVersion(HttpVersion.valueOf(dumpedResponse.getVersion()));
response.setStatus(HttpResponseStatus.valueOf(dumpedResponse.getStatusCode()));
for (Map.Entry<String, String> entry : dumpedResponse.getHeaders().entrySet()) {
response.headers().add(entry.getKey(), entry.getValue());
}
response.content().writeBytes(dumpedResponse.getContent().getBytes());
}
private Response failoverResponse(FullHttpRequest request) {
final Request dumpedRequest = MessageFactory.createRequest(request);
List<Session> sessions = restoreSessions(this.file);
final Optional<Session> session = tryFind(sessions, isForRequest(dumpedRequest));
if (session.isPresent()) {
return session.get().getResponse();
}
logger.error("No match request found: {}", dumpedRequest);
throw new RuntimeException("no failover response found");
}
private Predicate<Session> isForRequest(final Request dumpedRequest) {
return new Predicate<Session>() {
@Override
public boolean apply(Session session) {
return session.getRequest().match(dumpedRequest);
}
};
}
}
static {
VALUE_TYPE = TypeFactory.defaultInstance().uncheckedSimpleType(String.class);
instance = new StringArraySerializer();
}