@Override
protected Object defaultTypeDataFromBinary(final byte[] bytes) {
final ByteBuffer buf = ByteBuffer.wrap(bytes);
final int initialBytes = buf.getInt();
// temporary hack for backward compatibility
final boolean skipConfig = (initialBytes > 0);
final byte[] typeNameBytes = skipConfig ? new byte[initialBytes] : new byte[buf.getInt()];
final byte[] visibilityManagementClassNameBytes = new byte[buf.getInt()];
final byte[] attrBytes = skipConfig ? new byte[0] : new byte[buf.getInt()];
final byte[] encodedTypeBytes = new byte[buf.getInt()];
final byte[] adapterIdBytes = new byte[buf.getInt()];
buf.get(typeNameBytes);
buf.get(visibilityManagementClassNameBytes); // ignore...old release
buf.get(attrBytes);
buf.get(encodedTypeBytes);
buf.get(adapterIdBytes);
adapterId = new ByteArrayId(adapterIdBytes);
final String typeName = StringUtils.stringFromBinary(typeNameBytes);
final String encodedType = StringUtils.stringFromBinary(encodedTypeBytes);
try {
final SimpleFeatureType myType = DataUtilities.createType(typeName, encodedType);
featureType = myType;
return featureType;
} catch (final SchemaException e) {
LOGGER.error("Unable to deserialized feature type", e);
}
final SimpleFeatureUserDataConfigurationSet userDataConfiguration =
new SimpleFeatureUserDataConfigurationSet();
userDataConfiguration.addConfigurations(typeName, new TimeDescriptorConfiguration(featureType));
userDataConfiguration.addConfigurations(
typeName, new SimpleFeatureStatsConfigurationCollection(featureType));
userDataConfiguration.addConfigurations(
typeName, new SimpleFeaturePrimaryIndexConfiguration(featureType));
try {
userDataConfiguration.fromJsonString(StringUtils.stringFromBinary(attrBytes), featureType);
} catch (final IOException e) {
LOGGER.error("Failure to decode simple feature user data configuration", e);
}
return null;
}
@Override
protected byte[] defaultTypeDataToBinary() {
final String encodedType = DataUtilities.encodeType(featureType);
final String typeName = featureType.getTypeName();
final byte[] typeNameBytes = StringUtils.stringToBinary(typeName);
final byte[] encodedTypeBytes = StringUtils.stringToBinary(encodedType);
byte[] attrBytes = new byte[0];
final SimpleFeatureUserDataConfigurationSet userDataConfiguration =
new SimpleFeatureUserDataConfigurationSet();
userDataConfiguration.addConfigurations(typeName, new TimeDescriptorConfiguration());
userDataConfiguration.addConfigurations(
typeName, new SimpleFeatureStatsConfigurationCollection());
userDataConfiguration.addConfigurations(typeName, new SimpleFeaturePrimaryIndexConfiguration());
userDataConfiguration.configureFromType(featureType);
try {
attrBytes = StringUtils.stringToBinary(userDataConfiguration.asJsonString());
} catch (final IOException e) {
LOGGER.error("Failure to encode simple feature user data configuration", e);
}
final ByteBuffer buf =
ByteBuffer.allocate(
encodedTypeBytes.length
+ typeNameBytes.length
+ adapterId.getBytes().length
+ attrBytes.length
+ 24);
buf.putInt(0); // a signal for the new version
buf.putInt(typeNameBytes.length);
buf.putInt(0); // old visibility (backward compatibility)
buf.putInt(attrBytes.length);
buf.putInt(encodedTypeBytes.length);
buf.putInt(adapterId.getBytes().length);
buf.put(typeNameBytes);
buf.put(attrBytes);
buf.put(encodedTypeBytes);
buf.put(adapterId.getBytes());
return buf.array();
}
public WholeFeatureDataAdapter(final SimpleFeatureType featureType) {
super(
new ArrayList<
PersistentIndexFieldHandler<SimpleFeature, ? extends CommonIndexValue, Object>>(),
new ArrayList<NativeFieldHandler<SimpleFeature, Object>>(),
null,
featureType);
this.featureType = featureType;
adapterId = new ByteArrayId(StringUtils.stringToBinary(featureType.getTypeName()));
statsManager = new StatsManager(this, featureType, featureType, null);
}
@Override
public ByteArrayId getDataId(final SimpleFeature entry) {
return new ByteArrayId(StringUtils.stringToBinary(entry.getID()));
}
@Override
public String getId() {
return StringUtils.intToString(hashCode());
}
public class TextIndexStrategy implements FieldIndexStrategy<TextQueryConstraint, String> {
protected static final String START_END_MARKER = "\01";
protected static final byte[] START_END_MARKER_BYTE =
StringUtils.stringToBinary(START_END_MARKER);
private int start = 2;
private int end = 4;
public TextIndexStrategy() {
super();
}
public TextIndexStrategy(final int start, final int end) {
this();
this.start = start;
this.end = end;
}
@Override
public byte[] toBinary() {
final ByteBuffer bb = ByteBuffer.allocate(8);
bb.putInt(start);
bb.putInt(end);
return bb.array();
}
@Override
public void fromBinary(final byte[] bytes) {
final ByteBuffer bb = ByteBuffer.wrap(bytes);
start = bb.getInt();
end = bb.getInt();
}
@Override
public List<ByteArrayRange> getQueryRanges(
final TextQueryConstraint indexedRange, final IndexMetaData... hints) {
return indexedRange.getRange(start, end);
}
@Override
public List<ByteArrayRange> getQueryRanges(
final TextQueryConstraint indexedRange,
final int maxEstimatedRangeDecomposition,
final IndexMetaData... hints) {
return indexedRange.getRange(start, end);
}
@Override
public List<ByteArrayId> getInsertionIds(final List<FieldInfo<String>> indexedData) {
final List<ByteArrayId> insertionIds = new ArrayList<>();
for (final FieldInfo<String> fieldInfo : indexedData) {
insertionIds.addAll(
grams(
START_END_MARKER + fieldInfo.getDataValue().getValue() + START_END_MARKER,
start,
end));
}
return insertionIds;
}
@Override
public List<ByteArrayId> getInsertionIds(
final List<FieldInfo<String>> indexedData, final int maxEstimatedDuplicateIds) {
return getInsertionIds(indexedData);
}
@Override
public List<FieldInfo<String>> getRangeForId(final ByteArrayId insertionId) {
return Collections.emptyList();
}
@Override
public String getId() {
return "NGRAM_" + start + "_" + end;
}
protected static List<ByteArrayId> grams(final String value, final int start, final int end) {
final NGramTokenizer nGramTokenizer = new NGramTokenizer(start, end);
final List<ByteArrayId> tokens = new ArrayList<ByteArrayId>();
try {
nGramTokenizer.setReader(new StringReader(value));
final CharTermAttribute charTermAttribute =
nGramTokenizer.addAttribute(CharTermAttribute.class);
nGramTokenizer.reset();
while (nGramTokenizer.incrementToken()) {
tokens.add(new ByteArrayId(charTermAttribute.toString()));
}
nGramTokenizer.end();
nGramTokenizer.close();
} catch (final IOException e) {
e.printStackTrace();
}
return tokens;
}
public static final byte[] toIndexByte(final String ngram) {
return toIndexByte(StringUtils.stringToBinary(ngram));
}
public static final byte[] toIndexByte(final byte[] ngramBytes) {
final ByteBuffer buffer = ByteBuffer.allocate(ngramBytes.length + 4);
buffer.putInt(ngramBytes.length);
buffer.put(ngramBytes);
return buffer.array();
}
@Override
public Set<ByteArrayId> getNaturalSplits() {
return null;
}
@Override
public List<IndexMetaData> createMetaData() {
return Collections.emptyList();
}
}
public static final byte[] toIndexByte(final String ngram) {
return toIndexByte(StringUtils.stringToBinary(ngram));
}
@Test
public void testLocalityGroups() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
final String tableName = StringUtils.stringFromBinary(index.getId().getBytes());
final byte[] adapterId = adapter.getAdapterId().getBytes();
accumuloOptions.setUseLocalityGroups(false);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
try {
// as we are not using locality groups, we expect that this will
// return false
assertEquals(false, accumuloOperations.localityGroupExists(tableName, adapterId));
} catch (final AccumuloException | TableNotFoundException e) {
LOGGER.error("Locality Group check failed", e);
}
final TestGeometry geom1 =
(TestGeometry)
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
.next();
// of course, the point is actually stored in this case
assertEquals("test_pt_1", geom1.id);
}
accumuloOptions.setUseLocalityGroups(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId2 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
try {
// now that locality groups are turned on, we expect this to
// return
// true
assertEquals(true, accumuloOperations.localityGroupExists(tableName, adapterId));
} catch (final AccumuloException | TableNotFoundException e) {
LOGGER.error("Locality Group check failed", e);
}
final TestGeometry geom2 =
(TestGeometry)
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))
.next();
// of course, the point is actually stored in this case
assertEquals("test_pt_2", geom2.id);
}
}