@Override
public void deserialize(byte[] data, LuceneWorksBuilder hydrator) {
final ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
final int majorVersion = inputStream.read();
final int minorVersion = inputStream.read();
final Protocol protocol = protocols.getProtocol(majorVersion, minorVersion);
Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
GenericDatumReader<GenericRecord> reader =
new GenericDatumReader<>(protocol.getType("Message"));
GenericRecord result;
try {
result = reader.read(null, decoder);
} catch (IOException e) {
throw log.unableToDeserializeAvroStream(e);
}
classReferences = asListOfString(result, "classReferences");
final List<GenericRecord> operations = asListOfGenericRecords(result, "operations");
final ConversionContext conversionContext = new ContextualExceptionBridgeHelper();
for (GenericRecord operation : operations) {
String schema = operation.getSchema().getName();
if ("OptimizeAll".equals(schema)) {
hydrator.addOptimizeAll();
} else if ("PurgeAll".equals(schema)) {
hydrator.addPurgeAllLuceneWork(asClass(operation, "class"));
} else if ("Flush".equals(schema)) {
hydrator.addFlush();
} else if ("Delete".equals(schema)) {
processId(operation, hydrator);
hydrator.addDeleteLuceneWork(asClass(operation, "class"), conversionContext);
} else if ("DeleteByQuery".equals(schema)) {
String entityClassName = asClass(operation, "class");
int queryKey = asInt(operation, "key");
DeleteByQuerySupport.StringToQueryMapper mapper =
DeleteByQuerySupport.getStringToQueryMapper(queryKey);
List<Utf8> stringList = asListOfString(operation, "query");
String[] query = new String[stringList.size()];
for (int i = 0; i < stringList.size(); ++i) {
query[i] = stringList.get(i).toString();
}
hydrator.addDeleteByQueryLuceneWork(entityClassName, mapper.fromString(query));
} else if ("Add".equals(schema)) {
buildLuceneDocument(asGenericRecord(operation, "document"), hydrator);
Map<String, String> analyzers = getAnalyzers(operation);
processId(operation, hydrator);
hydrator.addAddLuceneWork(asClass(operation, "class"), analyzers, conversionContext);
} else if ("Update".equals(schema)) {
buildLuceneDocument(asGenericRecord(operation, "document"), hydrator);
Map<String, String> analyzers = getAnalyzers(operation);
processId(operation, hydrator);
hydrator.addUpdateLuceneWork(asClass(operation, "class"), analyzers, conversionContext);
} else {
throw log.cannotDeserializeOperation(schema);
}
}
}
@Test
public void testSplitProtocolBuild() {
Protocol p = new Protocol("P", null, "foo");
p.addProp("property", "some value");
String protocolString = p.toString();
final int mid = protocolString.length() / 2;
String[] parts = {
protocolString.substring(0, mid), protocolString.substring(mid),
};
Protocol parsedStringProtocol = org.apache.avro.Protocol.parse(protocolString);
Protocol parsedArrayOfStringProtocol =
org.apache.avro.Protocol.parse(
protocolString.substring(0, mid), protocolString.substring(mid));
assertNotNull(parsedStringProtocol);
assertNotNull(parsedArrayOfStringProtocol);
assertEquals(parsedStringProtocol.toString(), parsedArrayOfStringProtocol.toString());
}
@SuppressWarnings("all")
/** */
@org.apache.avro.specific.AvroGenerated
public interface G_UserSpaceDataTypes {
public static final org.apache.avro.Protocol PROTOCOL =
org.apache.avro.Protocol.parse(
"{\"protocol\":\"G_UserSpaceDataTypes\",\"namespace\":\"graphene.model.idl\",\"doc\":\"\",\"types\":[{\"type\":\"enum\",\"name\":\"G_PropertyTag\",\"doc\":\"* Tags are defined by the application layer as a taxonomy of user and\\r\\n\\t * application concepts, independent of the data sources. This allows\\r\\n\\t * application semantics to be re-used with new data, with a minimum of new\\r\\n\\t * software design and development. Data layer entity types, link types and\\r\\n\\t * properties should be mapped into the list of tags. The application layer\\r\\n\\t * must be able to search by native field name or by tag interchangeably,\\r\\n\\t * and properties returned must contain both native field names as well as\\r\\n\\t * tags.\\r\\n\\t * \\r\\n\\t * The list of tags may change as application features evolve, though that\\r\\n\\t * will require collaboration with the data layer providers. Evolving the\\r\\n\\t * tag list should not change the Data Access or Search APIs.\\r\\n\\t * \\r\\n\\t * This is the current list of tags for Properties:\\r\\n\\t * \\r\\n\\t *\",\"symbols\":[\"ID\",\"TYPE\",\"ENTITY_TYPE\",\"ACCOUNT_OWNER\",\"CLUSTER_SUMMARY\",\"NAME\",\"LABEL\",\"STAT\",\"TEXT\",\"STATUS\",\"ANNOTATION\",\"WARNING\",\"LINKED_DATA\",\"IMAGE\",\"VIDEO\",\"AUDIO\",\"GEO\",\"COUNTRY_CODE\",\"DATE\",\"AMOUNT\",\"INFLOWING\",\"OUTFLOWING\",\"COUNT\",\"SERIES\",\"CONSTRUCTED\",\"RAW\",\"USD\",\"DURATION\",\"REDACTED\"]},{\"type\":\"enum\",\"name\":\"G_CanonicalIndustryType\",\"doc\":\"* Industry types are for classifying datasets, data sources, or individual\\r\\n\\t * datum.\\r\\n\\t * \\r\\n\\t *\",\"symbols\":[\"REDACTED\",\"UNKNOWN_OR_UNSURE\",\"BANKING\",\"FINANCIAL_SERVICES\",\"FOOD_RESTAURANT\",\"COMMODITY_PROVIDER\",\"ISP\",\"TELCO\",\"TELEVISION\",\"EMAIL_PROVIDER\",\"MULTI\",\"TRANSPORT\",\"LOGISTICS\",\"ADVERTISEMENT_AGENCY\",\"EMPLOYMENT_AGENCY\",\"GOVERNMENT_AGENCY\",\"INTERNATIONAL_AGENCY\",\"NEWS_AGENCY\",\"POSTAL\",\"REAL_ESTATE\",\"EDUCATION\",\"HEALTHCARE\",\"CHARITY\"]},{\"type\":\"enum\",\"name\":\"G_EntityTag\",\"doc\":\"* The 21st century meaning, \\\"conceal from unauthorized\\r\\n\\t\\t\\t\\t\\t\\t * view; censor but do not destroy\\\"\",\"symbols\":[\"ACCOUNT_OWNER\",\"ACCOUNT\",\"GROUP\",\"CLUSTER\",\"CLUSTER_SUMMARY\",\"FILE\",\"ANONYMOUS\",\"OTHER\",\"REDACTED\"]},{\"type\":\"enum\",\"name\":\"G_LinkTag\",\"doc\":\"* The 21st century meaning, \\\"conceal from unauthorized\\r\\n\\t\\t\\t\\t\\t\\t * view; censor but do not destroy\\\"\",\"symbols\":[\"FINANCIAL\",\"SOCIAL\",\"COMMUNICATION\",\"EVENT\",\"OTHER\",\"REDACTED\",\"FATHER_OF\",\"SON_OF\",\"MOTHER_OF\",\"DAUGHTER_OF\",\"BROTHER_OF\",\"SISTER_OF\",\"GUARDIAN_OF\",\"KIN_OF\",\"COUSIN_OF\",\"WIFE_OF\",\"HUSBAND_OF\",\"EMPLOYEE_OF\",\"OWNER_OF\",\"SELLER_OF\",\"BUYER_OF\",\"CREATOR_OF\",\"FRIEND_OF\",\"ENEMY_OF\",\"KNOWS\"]},{\"type\":\"enum\",\"name\":\"G_PropertyType\",\"doc\":\"* The 21st century meaning, \\\"conceal from unauthorized\\r\\n\\t\\t\\t\\t\\t\\t * view; censor but do not destroy\\\"\",\"symbols\":[\"DOUBLE\",\"LONG\",\"BOOLEAN\",\"STRING\",\"DATE\",\"GEO\",\"IMAGE\",\"URI\",\"OTHER\",\"REDACTED\"]},{\"type\":\"enum\",\"name\":\"G_VisualTag\",\"doc\":\"* The 21st century meaning, \\\"conceal from unauthorized\\r\\n\\t\\t\\t\\t\\t\\t * view; censor but do not destroy\\\"\",\"symbols\":[\"CHOROPLETH\",\"CARTOGRAM\",\"DOT_DISTRIBUTION\",\"PROPORTIONAL_SYMBOL\",\"CONTOUR_MAP\",\"DASYMETRIC\",\"FLOW_MAP\",\"SOM\",\"TIME_LINE\",\"TIME_SERIES\",\"GANTT\",\"STREAM_GRAPH\",\"ARC_DIAGRAM\",\"POLAR_AREA\",\"SANKEY\",\"ALLUVIAL\",\"PIE_CHART\",\"HISTOGRAM\",\"TAG_CLOUD\",\"BUBBLE_CHART\",\"BAR_CHART\",\"TREE_MAP\",\"SCATTER_PLOT\",\"LINE_CHART\",\"STEP_CHART\",\"AREA_CHART\",\"HEAT_MAP\",\"SPIDER_CHART\",\"CANDLESTICK\",\"TREE_REGULAR\",\"TREE_RADIAL\",\"TREE_HYPERBOLIC\",\"RADIAL_HIERARCHY\",\"MATRIX\",\"NODE_LINK_DIAGRAM\",\"NODE_LINK_DIAGRAM3D\",\"CIRCULAR_HIERARCHY\",\"HIVE_PLOT\",\"SUBWAY_MAP\",\"DEFAULT\",\"HIDDEN\",\"IMAGE\",\"VIDEO\",\"INFINITE_SCROLL\",\"LIST\",\"PLUGIN\",\"REDACTED\"]},{\"type\":\"record\",\"name\":\"G_Provenance\",\"doc\":\"* This is a placeholder for future modeling of provenance. It is not a\\r\\n\\t * required field in any service calls.\",\"fields\":[{\"name\":\"uri\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Placeholder for now. Express provenance as a single URI.\"}]},{\"type\":\"record\",\"name\":\"G_Uncertainty\",\"doc\":\"* This is a placeholder for future modeling of uncertainty. It is not a\\r\\n\\t * required field in any service calls.\\r\\n\\t *\",\"fields\":[{\"name\":\"confidence\",\"type\":\"double\",\"doc\":\"* Placeholder for now. Express original source confidence as a single\\r\\n\\t\\t * number from 0 to 1.\",\"default\":1},{\"name\":\"type\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* For describing the nature of the uncertainty (source data, algorithm,\\r\\n\\t\\t * unit, etc)\",\"default\":null}]},{\"type\":\"record\",\"name\":\"G_LinkedData\",\"doc\":\"* A URL and MIME type representing a pointer to text, image or other\\r\\n\\t * external resource.\",\"fields\":[{\"name\":\"url\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"URL of the resource\"},{\"name\":\"mimeType\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"MIME type of the resource\",\"default\":null},{\"name\":\"title\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* description of the linked resource (suitable for display to the user\\r\\n\\t\\t * in an hypertext link)\",\"default\":null}]},{\"type\":\"record\",\"name\":\"G_GeoData\",\"doc\":\"* Structured representation of geo-spatial data.\",\"fields\":[{\"name\":\"text\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"an address or other place reference; unstructured text field\",\"default\":null},{\"name\":\"lat\",\"type\":[\"double\",\"null\"],\"doc\":\"latitude\",\"default\":null},{\"name\":\"lon\",\"type\":[\"double\",\"null\"],\"doc\":\"longitude\",\"default\":null},{\"name\":\"cc\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"ISO 3 digit country code\",\"default\":null}]},{\"type\":\"enum\",\"name\":\"G_ContinentCode\",\"doc\":\"* Standard two letter continent code\\r\\n\\t *\",\"symbols\":[\"AF\",\"AS\",\"EU\",\"NA\",\"SA\",\"OC\",\"AN\"]},{\"type\":\"record\",\"name\":\"G_Country\",\"doc\":\"* Structured representation of country data, which includes geo-spatial\\r\\n\\t * data.\\r\\n\\t *\",\"fields\":[{\"name\":\"country\",\"type\":\"G_GeoData\",\"doc\":\"country geo data, including the name as text\"},{\"name\":\"region\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"* the name of the global region to which the country belongs; any\\r\\n\\t\\t * common classification here is acceptable\"},{\"name\":\"continent\",\"type\":\"G_ContinentCode\",\"doc\":\"continent\"}]},{\"type\":\"enum\",\"name\":\"G_DateInterval\",\"doc\":\"* Temporal resolution of a duration\\r\\n\\t *\",\"symbols\":[\"SECONDS\",\"HOURS\",\"DAYS\",\"WEEKS\",\"MONTHS\",\"QUARTERS\",\"YEARS\"]},{\"type\":\"record\",\"name\":\"G_Duration\",\"doc\":\"* A temporal duration\\r\\n\\t *\",\"fields\":[{\"name\":\"interval\",\"type\":\"G_DateInterval\",\"doc\":\"time aggregation level, e.g. use monthly data\"},{\"name\":\"numIntervals\",\"type\":\"long\",\"doc\":\"number of intervals, e.g. 12 monthly intervals is a 1 year duration\"}]},{\"type\":\"record\",\"name\":\"G_DateRange\",\"doc\":\"* Describes a date range at a specific resolution.\\r\\n\\t *\",\"fields\":[{\"name\":\"startDate\",\"type\":\"long\"},{\"name\":\"numBins\",\"type\":\"long\",\"doc\":\"number of bins to return, e.g. 12 monthly bins for 1 year of data\"},{\"name\":\"durationPerBin\",\"type\":\"G_Duration\",\"doc\":\"* number of intervals in a bin, e.g. 2 months/bin in 12 bins for 2\\r\\n\\t\\t * years of data\"}]},{\"type\":\"enum\",\"name\":\"G_RangeType\",\"doc\":\"* Allowed types for Ranges of values.\\r\\n\\t *\",\"symbols\":[\"SINGLETON\",\"LIST\",\"BOUNDED\",\"DISTRIBUTION\"]},{\"type\":\"enum\",\"name\":\"G_Gender\",\"doc\":\"* Allowed types for Gender values.\\r\\n\\t *\",\"symbols\":[\"REDACTED\",\"MALE\",\"FEMALE\",\"UKNOWN\",\"OTHER\"]},{\"type\":\"record\",\"name\":\"G_SingletonRange\",\"doc\":\"* Single value\\r\\n\\t *\",\"fields\":[{\"name\":\"value\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"int\",\"float\",\"double\",\"long\",\"boolean\",\"G_GeoData\"]},{\"name\":\"type\",\"type\":\"G_PropertyType\",\"doc\":\"One of DOUBLE, LONG, BOOLEAN, STRING, DATE, GEO, OTHER\"}]},{\"type\":\"record\",\"name\":\"G_ListRange\",\"doc\":\"* List of values\\r\\n\\t *\",\"fields\":[{\"name\":\"values\",\"type\":{\"type\":\"array\",\"items\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"int\",\"float\",\"double\",\"long\",\"boolean\",\"G_GeoData\"]}},{\"name\":\"type\",\"type\":\"G_PropertyType\",\"doc\":\"One of DOUBLE, LONG, BOOLEAN, STRING, DATE, GEO, OTHER\"}]},{\"type\":\"record\",\"name\":\"G_BoundedRange\",\"doc\":\"* Bounded or unbounded range values\\r\\n\\t *\",\"fields\":[{\"name\":\"start\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"int\",\"float\",\"double\",\"long\",\"boolean",
"\",\"G_GeoData\",\"null\"],\"doc\":\"start of range, or null if unbounded start\"},{\"name\":\"end\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"int\",\"float\",\"double\",\"long\",\"boolean\",\"G_GeoData\",\"null\"],\"doc\":\"end of range, or null if unbounded start\"},{\"name\":\"inclusive\",\"type\":\"boolean\",\"doc\":\"* If true, range includes specified endpoint. If false, range is\\r\\n\\t\\t * exclusive.\"},{\"name\":\"type\",\"type\":\"G_PropertyType\",\"doc\":\"One of DOUBLE, LONG, BOOLEAN, STRING, DATE, GEO, OTHER\"}]},{\"type\":\"record\",\"name\":\"G_Frequency\",\"doc\":\"* A frequency or probability element of a distribution.\\r\\n\\t *\",\"fields\":[{\"name\":\"range\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"int\",\"float\",\"double\",\"long\",\"boolean\",\"G_GeoData\",\"G_ListRange\",\"G_BoundedRange\"],\"doc\":\"the value range which occurs with some specified frequency\"},{\"name\":\"frequency\",\"type\":\"double\",\"doc\":\"frequency as a count, or probability as a value from 0-1.\"}]},{\"type\":\"record\",\"name\":\"G_DistributionRange\",\"doc\":\"* Describes a distribution of values.\\r\\n\\t *\",\"fields\":[{\"name\":\"distribution\",\"type\":{\"type\":\"array\",\"items\":\"G_Frequency\"}},{\"name\":\"rangeType\",\"type\":\"G_RangeType\",\"doc\":\"Describes how the values in the distribution are summarized\"},{\"name\":\"type\",\"type\":\"G_PropertyType\",\"doc\":\"* The type of value that the distribution describes. One of DOUBLE,\\r\\n\\t\\t * LONG, BOOLEAN, STRING, DATE, GEO, OTHER\"},{\"name\":\"isProbability\",\"type\":\"boolean\",\"doc\":\"True if a probability distribution, false if a frequency distribution\",\"default\":false}]},{\"type\":\"record\",\"name\":\"G_Property\",\"doc\":\"* Each property on an Entity or Link is a name-value pair, with data type\\r\\n\\t * information, as well as optional provenance. Tags provide a way for the\\r\\n\\t * data provider to associate semantic annotations to each property in terms\\r\\n\\t * of the semantics of the application.\\r\\n\\t *\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"the field name in the underlying data source\"},{\"name\":\"friendlyText\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user-friendly short-text for key (displayable)\",\"default\":null},{\"name\":\"range\",\"type\":[\"G_SingletonRange\",\"G_ListRange\",\"G_BoundedRange\",\"G_DistributionRange\"],\"doc\":\"range of values\",\"default\":null},{\"name\":\"provenance\",\"type\":[\"G_Provenance\",\"null\"],\"default\":null},{\"name\":\"uncertainty\",\"type\":[\"G_Uncertainty\",\"null\"],\"default\":null},{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"G_PropertyTag\"},\"doc\":\"* one or more tags from the Tag list, used to map this source-specific\\r\\n\\t\\t * field into the semantics of applications\"}]},{\"type\":\"record\",\"name\":\"G_DataSetField\",\"doc\":\"* A field within a DataSet, which in turn is a real or virtual table within\\r\\n\\t * a DataSource. From Non Avro version by PWG.\",\"fields\":[{\"name\":\"name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"name of the field\"},{\"name\":\"friendlyName\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user-friendly short-text for key (displayable)\",\"default\":null},{\"name\":\"type\",\"type\":\"G_PropertyType\",\"doc\":\"One of DOUBLE, LONG, BOOLEAN, STRING, DATE, GEO, OTHER\"},{\"name\":\"sortable\",\"type\":\"boolean\",\"default\":true},{\"name\":\"searchable\",\"type\":\"boolean\",\"default\":true},{\"name\":\"reportable\",\"type\":\"boolean\",\"default\":true}]},{\"type\":\"record\",\"name\":\"G_DataSet\",\"fields\":[{\"name\":\"name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"name of the dataset\"},{\"name\":\"isEntity\",\"type\":\"boolean\",\"default\":true},{\"name\":\"isTransaction\",\"type\":\"boolean\",\"default\":true},{\"name\":\"fields\",\"type\":{\"type\":\"array\",\"items\":\"G_DataSetField\"}}]},{\"type\":\"record\",\"name\":\"G_DataSource\",\"fields\":[{\"name\":\"id\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"id of the data source\"},{\"name\":\"name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"name of the data source\"},{\"name\":\"friendlyName\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user-friendly short-text for key (displayable)\",\"default\":null},{\"name\":\"isEntity\",\"type\":\"boolean\",\"default\":true},{\"name\":\"isTransaction\",\"type\":\"boolean\",\"default\":true},{\"name\":\"dataSets\",\"type\":{\"type\":\"array\",\"items\":\"G_DataSet\"}}]},{\"type\":\"record\",\"name\":\"G_DataSourceList\",\"fields\":[{\"name\":\"dataSources\",\"type\":{\"type\":\"array\",\"items\":\"G_DataSource\"}}]},{\"type\":\"record\",\"name\":\"G_Entity\",\"doc\":\"* The nodes in the social, financial, communications or other graphs. May\\r\\n\\t * represent concrete individuals or organizations, specific proxies such as\\r\\n\\t * accounts, or the implicit individuals or groups behind those other\\r\\n\\t * entities.\",\"fields\":[{\"name\":\"uid\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"* This uid must represent either (1) a globally unique identifier that\\r\\n\\t\\t * can be used to retrieve data for an explicit entity, or (2) encoded\\r\\n\\t\\t * query information that can be used to find a set of associated record\\r\\n\\t\\t * notionally representing an implicit entity (e.g.\\r\\n\\t\\t * Loans&FirstName&LastName&Gender). Must not be used for IDs that\\r\\n\\t\\t * aren't globally unique. For example, in Kiva, \\\"Lenders\\\" has a UID\\r\\n\\t\\t * (\\\"L12345\\\") while \\\"Borrowers\\\" have an encoded search in the Loans\\r\\n\\t\\t * table for uid (\\\"B{loan:23456;name=Daniel}\\\"). The encoded information\\r\\n\\t\\t * is data layer-specific, may be different from entity to entity or\\r\\n\\t\\t * data set to data set, and should be considered opaque to the\\r\\n\\t\\t * consumers of the entities. Entities of type 2 should always have the\\r\\n\\t\\t * Entity Tag ANONYMOUS to help distinguish them when required.\"},{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"G_EntityTag\"},\"doc\":\"Entity Tags (see above, e.g. \\\"ACCOUNT\\\")\"},{\"name\":\"provenance\",\"type\":[\"G_Provenance\",\"null\"],\"default\":null},{\"name\":\"uncertainty\",\"type\":[\"G_Uncertainty\",\"null\"],\"default\":null},{\"name\":\"properties\",\"type\":{\"type\":\"array\",\"items\":\"G_Property\"}}]},{\"type\":\"record\",\"name\":\"G_Link\",\"doc\":\"* The links in the social, financial, communications or other graphs. May\\r\\n\\t * represent communication events, financial transactions or social\\r\\n\\t * connections.\",\"fields\":[{\"name\":\"source\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"source entity uid\",\"default\":null},{\"name\":\"target\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"target entity uid\",\"default\":null},{\"name\":\"directed\",\"type\":\"boolean\",\"doc\":\"true if directed, false if undirected\",\"default\":true},{\"name\":\"provenance\",\"type\":[\"G_Provenance\",\"null\"],\"default\":null},{\"name\":\"uncertainty\",\"type\":[\"G_Uncertainty\",\"null\"],\"default\":null},{\"name\":\"properties\",\"type\":{\"type\":\"array\",\"items\":\"G_Property\"}},{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"G_LinkTag\"},\"doc\":\"Link Tags\"}]},{\"type\":\"record\",\"name\":\"G_Cluster\",\"doc\":\"* Cluster of nodes in the social, financial, communications or other\\r\\n\\t * graphs.\",\"fields\":[{\"name\":\"uid\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"* A unique identifier use to retrieve data about this cluster. Should\\r\\n\\t\\t * not be used for non-global identifiers.\"},{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"G_EntityTag\"},\"doc\":\"Entity Tags (see DataTypes, e.g. \\\"CLUSTER\\\")\"},{\"name\":\"provenance\",\"type\":[\"G_Provenance\",\"null\"],\"default\":null},{\"name\":\"uncertainty\",\"type\":[\"G_Uncertainty\",\"null\"],\"default\":null},{\"name\":\"properties\",\"type\":{\"type\":\"array\",\"items\":\"G_Property\"}},{\"name\":\"members\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\"doc\":\"List of id's of the members of this cluster\"},{\"name\":\"subclusters\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\"doc\":\"* List of id's of the subclusters of this cluster - empty if this is a\\r\\n\\t\\t * leaf cluster in cluster hierarchy\"},{\"name\":\"parent\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* The id of the parent cluster if this is a cluster in a hierarchy -\\r\\n\\t\\t * Can be null if this is a root cluster *\",\"default\":null},{\"name\":\"root\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* The id of the root cluster (top level cluster) if this is a cluster\\",
"r\\n\\t\\t * in a hierarchy - Can be null if this is a root cluster *\",\"default\":null},{\"name\":\"level\",\"type\":\"int\",\"doc\":\"* The degree from the root this cluster is in the hierarchy - level = 0\\r\\n\\t\\t * if this is a root cluster *\",\"default\":0}]},{\"type\":\"record\",\"name\":\"G_Future\",\"doc\":\"* Represents the future results of an asynchronous task. Can be passed into\\r\\n\\t * the FutureResults service API\",\"fields\":[{\"name\":\"uid\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"unique id of this task\"},{\"name\":\"label\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"short human-readable description of task for display\"},{\"name\":\"service\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"uid of the service task is running on\",\"default\":null},{\"name\":\"started\",\"type\":\"long\",\"doc\":\"date/time task was started\"},{\"name\":\"completed\",\"type\":\"long\",\"doc\":\"date/time task was completed (negative if not completed yet)\",\"default\":-1}]},{\"type\":\"record\",\"name\":\"G_Service\",\"doc\":\"* Selectable services, returned by getServices() in various APIs\",\"fields\":[{\"name\":\"uid\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"unique id of this service\"},{\"name\":\"label\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"short human-readable description of service for display\"}]},{\"type\":\"record\",\"name\":\"G_AppInfo\",\"doc\":\"* Selectable services, returned by getAppInfo() in various meta APIs\",\"fields\":[{\"name\":\"appName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"unique id of this application\"},{\"name\":\"appVersion\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"version of this application\"},{\"name\":\"serverTime\",\"type\":\"long\",\"doc\":\"version of this application\"}]},{\"type\":\"record\",\"name\":\"G_Role\",\"doc\":\"* This is the role object used in security concerns. A G_User has one or more G_Role.\\r\\n\\t *\",\"fields\":[{\"name\":\"name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"* Name of the role\",\"default\":null},{\"name\":\"description\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* For describing the nature of the role\",\"default\":null}]},{\"type\":\"record\",\"name\":\"G_Permission\",\"doc\":\"* This is the permission object used in security concerns. A G_Role has zero or more G_Permissions\\r\\n\\t *\",\"fields\":[{\"name\":\"name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"* Name of the permission\",\"default\":null},{\"name\":\"description\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* For describing the nature of the permission\",\"default\":null}]},{\"type\":\"enum\",\"name\":\"G_UserFields\",\"symbols\":[\"active\",\"username\",\"avatar\",\"email\",\"fullname\",\"hashedpassword\",\"lastlogin\",\"lastmodified\",\"accountcreated\",\"numberlogins\",\"provenance\",\"uncertainty\"]},{\"type\":\"record\",\"name\":\"G_User\",\"fields\":[{\"name\":\"active\",\"type\":\"boolean\",\"doc\":\"\",\"default\":true},{\"name\":\"username\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"default\":\"notset\"},{\"name\":\"avatar\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"* some id of an avatar (could be a file name or other string to look\\r\\n\\t\\t * up)\",\"default\":null},{\"name\":\"email\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user's email\",\"default\":null},{\"name\":\"fullname\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user's full name\",\"default\":null},{\"name\":\"hashedpassword\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"doc\":\"user's hashed password\",\"default\":null},{\"name\":\"lastlogin\",\"type\":[\"long\",\"null\"],\"doc\":\"user's lastlogin datetime\",\"default\":0},{\"name\":\"lastmodified\",\"type\":[\"long\",\"null\"],\"doc\":\"user's modified datetime\",\"default\":0},{\"name\":\"accountcreated\",\"type\":[\"long\",\"null\"],\"doc\":\"user's account created datetime\",\"default\":0},{\"name\":\"numberlogins\",\"type\":\"int\",\"doc\":\"user's numberlogins\",\"default\":0}]},{\"type\":\"enum\",\"name\":\"G_GroupFields\",\"symbols\":[\"groupname\"]},{\"type\":\"record\",\"name\":\"G_Group\",\"fields\":[{\"name\":\"groupname\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"default\":\"notset\"}]},{\"type\":\"enum\",\"name\":\"G_WorkspaceFields\",\"symbols\":[\"active\",\"creatorusername\",\"json\",\"title\",\"workspaceid\",\"lastmodified\",\"createddate\"]},{\"type\":\"record\",\"name\":\"G_Workspace\",\"fields\":[{\"name\":\"active\",\"type\":\"boolean\",\"default\":true},{\"name\":\"creatorusername\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"default\":null},{\"name\":\"json\",\"type\":[{\"type\":\"string\",\"avro.java.string\":\"String\"},\"null\"],\"default\":null},{\"name\":\"title\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"default\":\"untitled\"},{\"name\":\"workspaceid\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"lastmodified\",\"type\":[\"long\",\"null\"],\"doc\":\"workspace's modified datetime\",\"default\":0},{\"name\":\"createddate\",\"type\":[\"long\",\"null\"],\"doc\":\"workspace's created datetime\",\"default\":0}]}],\"messages\":{}}");
@SuppressWarnings("all")
/** */
public interface Callback extends G_UserSpaceDataTypes {
public static final org.apache.avro.Protocol PROTOCOL =
graphene.model.idl.G_UserSpaceDataTypes.PROTOCOL;
}
}
public interface Sender<M extends Writable> {
public static final org.apache.avro.Protocol PROTOCOL =
org.apache.avro.Protocol.parse(
"{\"protocol\":\"Sender\",\"namespace\":\"de.jungblut.avro\",\"types\":[{\"type\":\"record\",\"name\":\"AvroBSPMessageBundle\",\"fields\":[{\"name\":\"data\",\"type\":\"bytes\"}]}],\"messages\":{\"transfer\":{\"request\":[{\"name\":\"messagebundle\",\"type\":\"AvroBSPMessageBundle\"}],\"response\":\"null\"}}}");
java.lang.Void transfer(AvroBSPMessageBundle<M> messagebundle)
throws org.apache.avro.AvroRemoteException;
@SuppressWarnings("all")
public interface Callback extends Sender {
public static final org.apache.avro.Protocol PROTOCOL = Sender.PROTOCOL;
public void transfer(
AvroBSPMessageBundle messagebundle, org.apache.avro.ipc.Callback<java.lang.Void> callback)
throws java.io.IOException;
}
}
@Test
public void experimentWithAvro() throws Exception {
String root = "org/hibernate/search/remote/codex/avro/v1_1/";
parseSchema(root + "attribute/TokenTrackingAttribute.avro", "attribute/TokenTrackingAttribute");
parseSchema(root + "attribute/CharTermAttribute.avro", "attribute/CharTermAttribute");
parseSchema(root + "attribute/PayloadAttribute.avro", "attribute/PayloadAttribute");
parseSchema(root + "attribute/KeywordAttribute.avro", "attribute/KeywordAttribute");
parseSchema(
root + "attribute/PositionIncrementAttribute.avro", "attribute/PositionIncrementAttribute");
parseSchema(root + "attribute/FlagsAttribute.avro", "attribute/FlagsAttribute");
parseSchema(root + "attribute/TypeAttribute.avro", "attribute/TypeAttribute");
parseSchema(root + "attribute/OffsetAttribute.avro", "attribute/OffsetAttribute");
parseSchema(root + "field/TermVector.avro", "field/TermVector");
parseSchema(root + "field/Index.avro", "field/Index");
parseSchema(root + "field/Store.avro", "field/Store");
parseSchema(root + "field/TokenStreamField.avro", "field/TokenStreamField");
parseSchema(root + "field/ReaderField.avro", "field/ReaderField");
parseSchema(root + "field/StringField.avro", "field/StringField");
parseSchema(root + "field/BinaryField.avro", "field/BinaryField");
parseSchema(root + "field/NumericIntField.avro", "field/NumericIntField");
parseSchema(root + "field/NumericLongField.avro", "field/NumericLongField");
parseSchema(root + "field/NumericFloatField.avro", "field/NumericFloatField");
parseSchema(root + "field/NumericDoubleField.avro", "field/NumericDoubleField");
parseSchema(root + "field/CustomFieldable.avro", "field/CustomFieldable");
parseSchema(root + "Document.avro", "Document");
parseSchema(root + "operation/Id.avro", "operation/Id");
parseSchema(root + "operation/OptimizeAll.avro", "operation/OptimizeAll");
parseSchema(root + "operation/PurgeAll.avro", "operation/PurgeAll");
parseSchema(root + "operation/Flush.avro", "operation/Flush");
parseSchema(root + "operation/Delete.avro", "operation/Delete");
parseSchema(root + "operation/Add.avro", "operation/Add");
parseSchema(root + "operation/Update.avro", "operation/Update");
parseSchema(root + "Message.avro", "Message");
String filename = root + "Works.avpr";
Protocol protocol = parseProtocol(filename, "Works");
final Schema termVectorSchema = protocol.getType("TermVector");
final Schema indexSchema = protocol.getType("Index");
final Schema storeSchema = protocol.getType("Store");
final Schema tokenTrackingAttribute = protocol.getType("TokenTrackingAttribute");
final Schema tokenStreamSchema = protocol.getType("TokenStreamField");
final Schema readerSchema = protocol.getType("ReaderField");
final Schema stringSchema = protocol.getType("StringField");
final Schema binarySchema = protocol.getType("BinaryField");
final Schema intFieldSchema = protocol.getType("NumericIntField");
final Schema longFieldSchema = protocol.getType("NumericLongField");
final Schema floatFieldSchema = protocol.getType("NumericFloatField");
final Schema doubleFieldSchema = protocol.getType("NumericDoubleField");
final Schema custonFieldableSchema = protocol.getType("CustomFieldable");
final Schema documentSchema = protocol.getType("Document");
final Schema idSchema = protocol.getType("Id");
final Schema optimizeAllSchema = protocol.getType("OptimizeAll");
final Schema purgeAllSchema = protocol.getType("PurgeAll");
final Schema flushSchema = protocol.getType("Flush");
final Schema deleteSchema = protocol.getType("Delete");
final Schema addSchema = protocol.getType("Add");
final Schema updateSchema = protocol.getType("Update");
Schema messageSchema = protocol.getType("Message");
final ByteArrayOutputStream out = new ByteArrayOutputStream();
GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(messageSchema);
Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
byte[] serializableSample = new byte[10];
for (int i = 0; i < 10; i++) {
serializableSample[i] = (byte) i;
}
List<String> classReferences = new ArrayList<String>();
classReferences.add(AvroTest.class.getName());
List<GenericRecord> fieldables = new ArrayList<GenericRecord>(1);
// custom fieldable
GenericRecord customFieldable = new GenericData.Record(custonFieldableSchema);
customFieldable.put("instance", ByteBuffer.wrap(serializableSample));
fieldables.add(customFieldable);
// numeric fields
GenericRecord numericField = createNumeric(intFieldSchema);
numericField.put("value", 3);
fieldables.add(numericField);
numericField = createNumeric(longFieldSchema);
numericField.put("value", 3l);
fieldables.add(numericField);
numericField = createNumeric(floatFieldSchema);
numericField.put("value", 2.3f);
fieldables.add(numericField);
numericField = createNumeric(doubleFieldSchema);
numericField.put("value", 2.3d);
fieldables.add(numericField);
// fields
GenericRecord field = createField(binarySchema);
field.put("offset", 0);
field.put("length", 10);
field.put("value", ByteBuffer.wrap(serializableSample));
fieldables.add(field);
field = createField(stringSchema);
field.put("value", stringSchema.getName());
field.put("store", "YES");
field.put("index", "ANALYZED");
field.put("termVector", "WITH_OFFSETS");
fieldables.add(field);
field = createField(tokenStreamSchema);
List<List<Object>> tokens = new ArrayList<List<Object>>();
List<Object> attrs = new ArrayList<Object>();
tokens.add(attrs);
GenericData.Record attr = new GenericData.Record(tokenTrackingAttribute);
List<Integer> positions = new ArrayList<Integer>();
positions.add(1);
positions.add(2);
positions.add(3);
positions.add(4);
attr.put("positions", positions);
attrs.add(attr);
attrs.add(ByteBuffer.wrap(serializableSample));
field.put("value", tokens);
field.put("termVector", "WITH_OFFSETS");
fieldables.add(field);
field = createField(readerSchema);
field.put("value", ByteBuffer.wrap(serializableSample));
field.put("termVector", "WITH_OFFSETS");
fieldables.add(field);
GenericRecord doc = new GenericData.Record(documentSchema);
doc.put("boost", 2.3f);
doc.put("fieldables", fieldables);
GenericRecord add = new GenericData.Record(addSchema);
add.put("class", classReferences.indexOf(AvroTest.class.getName()));
GenericRecord id = new GenericData.Record(idSchema);
id.put("value", ByteBuffer.wrap(serializableSample));
add.put("id", id);
add.put("document", doc);
Map<String, String> analyzers = new HashMap<String, String>();
analyzers.put("name", "ngram");
analyzers.put("description", "porter");
add.put("fieldToAnalyzerMap", analyzers);
GenericRecord delete = new GenericData.Record(deleteSchema);
delete.put("class", classReferences.indexOf(AvroTest.class.getName()));
id = new GenericData.Record(idSchema);
id.put("value", new Long(30));
delete.put("id", id);
GenericRecord purgeAll = new GenericData.Record(purgeAllSchema);
purgeAll.put("class", classReferences.indexOf(AvroTest.class.getName()));
GenericRecord optimizeAll = new GenericData.Record(optimizeAllSchema);
GenericRecord flush = new GenericData.Record(flushSchema);
List<GenericRecord> operations = new ArrayList<GenericRecord>(1);
operations.add(purgeAll);
operations.add(optimizeAll);
operations.add(flush);
operations.add(delete);
operations.add(add);
GenericRecord message = new GenericData.Record(messageSchema);
message.put("classReferences", classReferences);
message.put("operations", operations);
writer.write(message, encoder);
encoder.flush();
ByteArrayInputStream inputStream = new ByteArrayInputStream(out.toByteArray());
Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(messageSchema);
while (true) {
try {
GenericRecord result = reader.read(null, decoder);
System.out.println(result);
assertThat(result).isNotNull();
// operations
assertThat(result.get("operations")).isNotNull().isInstanceOf(List.class);
List<?> ops = (List<?>) result.get("operations");
assertThat(ops).hasSize(5);
// Flush
assertThat(ops.get(2)).isInstanceOf(GenericRecord.class);
GenericRecord flushOp = (GenericRecord) ops.get(2);
assertThat(flushOp.getSchema().getName()).isEqualTo("Flush");
// Delete
assertThat(ops.get(3)).isInstanceOf(GenericRecord.class);
GenericRecord deleteOp = (GenericRecord) ops.get(3);
assertThat(deleteOp.getSchema().getName()).isEqualTo("Delete");
Object actual = ((GenericRecord) deleteOp.get("id")).get("value");
assertThat(actual).isInstanceOf(Long.class);
assertThat(actual).isEqualTo(Long.valueOf(30));
// Add
assertThat(ops.get(4)).isInstanceOf(GenericRecord.class);
GenericRecord addOp = (GenericRecord) ops.get(4);
assertThat(addOp.getSchema().getName()).isEqualTo("Add");
actual = ((GenericRecord) addOp.get("id")).get("value");
assertThat(actual).isInstanceOf(ByteBuffer.class);
ByteBuffer bb = (ByteBuffer) actual;
assertThat(bb.hasArray()).isTrue();
byte[] copy = new byte[bb.remaining()];
bb.get(copy);
assertThat(serializableSample).isEqualTo(copy);
// fieldToAnalyzerMap
assertThat(addOp.get("fieldToAnalyzerMap")).isInstanceOf(Map.class);
assertThat((Map) addOp.get("fieldToAnalyzerMap")).hasSize(2);
// document
assertThat(addOp.get("document")).isNotNull();
GenericRecord document = (GenericRecord) addOp.get("document");
assertThat(document.get("boost")).isEqualTo(2.3f);
// numeric fields
assertThat(document.get("fieldables")).isNotNull().isInstanceOf(List.class);
List<?> fields = (List<?>) document.get("fieldables");
assertThat(fields).hasSize(9); // custom + 4 numerics + 4 fields
field = (GenericRecord) fields.get(0);
assertThat(field.getSchema().getName()).isEqualTo("CustomFieldable");
field = (GenericRecord) fields.get(1);
assertThat(field.getSchema().getName()).isEqualTo("NumericIntField");
assertThat(field.get("value")).isEqualTo(3);
assertNumericField(field);
field = (GenericRecord) fields.get(2);
assertThat(field.getSchema().getName()).isEqualTo("NumericLongField");
assertThat(field.get("value")).isEqualTo(3l);
assertNumericField(field);
field = (GenericRecord) fields.get(3);
assertThat(field.getSchema().getName()).isEqualTo("NumericFloatField");
assertThat(field.get("value")).isEqualTo(2.3f);
assertNumericField(field);
field = (GenericRecord) fields.get(4);
assertThat(field.getSchema().getName()).isEqualTo("NumericDoubleField");
assertThat(field.get("value")).isEqualTo(2.3d);
assertNumericField(field);
// fields
field = (GenericRecord) fields.get(5);
assertThat(field.getSchema().getName()).isEqualTo("BinaryField");
assertThat(field.get("value")).isInstanceOf(ByteBuffer.class);
assertField(field);
field = (GenericRecord) fields.get(6);
assertThat(field.getSchema().getName()).isEqualTo("StringField");
assertThat(field.get("value")).isInstanceOf(Utf8.class);
assertTermVector(field);
assertIndexAndStore(field);
assertField(field);
field = (GenericRecord) fields.get(7);
assertThat(field.getSchema().getName()).isEqualTo("TokenStreamField");
assertThat(field.get("value")).isInstanceOf(List.class);
List<List<Object>> l1 = (List<List<Object>>) field.get("value");
assertThat(l1.get(0)).as("Wrong attribute impl list").hasSize(2);
Object object = l1.get(0).get(0);
assertThat(object).isNotNull();
assertTermVector(field);
assertField(field);
field = (GenericRecord) fields.get(8);
assertThat(field.getSchema().getName()).isEqualTo("ReaderField");
assertThat(field.get("value")).isInstanceOf(ByteBuffer.class);
assertTermVector(field);
assertField(field);
} catch (EOFException eof) {
break;
} catch (Exception ex) {
ex.printStackTrace();
throw ex;
}
}
}