public void testParseAsBytesArray() throws Exception {
String path = "/org/elasticsearch/ingest/attachment/test/sample-files/text-in-english.txt";
byte[] bytes;
try (InputStream is = AttachmentProcessorTests.class.getResourceAsStream(path)) {
bytes = IOUtils.toByteArray(is);
}
Map<String, Object> document = new HashMap<>();
document.put("source_field", bytes);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> attachmentData =
(Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(
attachmentData.keySet(),
containsInAnyOrder("language", "content", "content_type", "content_length"));
assertThat(attachmentData.get("language"), is("en"));
assertThat(
attachmentData.get("content"),
is("\"God Save the Queen\" (alternatively \"God Save the King\""));
assertThat(attachmentData.get("content_type").toString(), containsString("text/plain"));
assertThat(attachmentData.get("content_length"), is(notNullValue()));
}
public void testCountry() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
GeoIpProcessor processor =
new GeoIpProcessor(
randomAsciiOfLength(10),
"source_field",
new DatabaseReader.Builder(database).build(),
"target_field",
EnumSet.allOf(GeoIpProcessor.Property.class));
Map<String, Object> document = new HashMap<>();
document.put("source_field", "82.170.213.79");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("82.170.213.79"));
@SuppressWarnings("unchecked")
Map<String, Object> geoData =
(Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(4));
assertThat(geoData.get("ip"), equalTo("82.170.213.79"));
assertThat(geoData.get("country_iso_code"), equalTo("NL"));
assertThat(geoData.get("country_name"), equalTo("Netherlands"));
assertThat(geoData.get("continent_name"), equalTo("Europe"));
}
public void testCity() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
GeoIpProcessor processor =
new GeoIpProcessor(
randomAsciiOfLength(10),
"source_field",
new DatabaseReader.Builder(database).build(),
"target_field",
EnumSet.allOf(GeoIpProcessor.Property.class));
Map<String, Object> document = new HashMap<>();
document.put("source_field", "8.8.8.8");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
@SuppressWarnings("unchecked")
Map<String, Object> geoData =
(Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(8));
assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
assertThat(geoData.get("country_iso_code"), equalTo("US"));
assertThat(geoData.get("country_name"), equalTo("United States"));
assertThat(geoData.get("continent_name"), equalTo("North America"));
assertThat(geoData.get("region_name"), equalTo("California"));
assertThat(geoData.get("city_name"), equalTo("Mountain View"));
assertThat(geoData.get("timezone"), equalTo("America/Los_Angeles"));
Map<String, Object> location = new HashMap<>();
location.put("lat", 37.386d);
location.put("lon", -122.0838d);
assertThat(geoData.get("location"), equalTo(location));
}
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
Map<String, Object> matches = grok.captures(fieldValue);
if (matches != null) {
matches.forEach((k, v) -> ingestDocument.setFieldValue(k, v));
} else {
throw new IllegalArgumentException(
"Grok expression does not match field value: [" + fieldValue + "]");
}
}
private Map<String, Object> parseDocument(String file, AttachmentProcessor processor)
throws Exception {
Map<String, Object> document = new HashMap<>();
document.put("source_field", getAsBase64(file));
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> attachmentData =
(Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
return attachmentData;
}
/**
* Returns a randomly selected existing field name out of the fields that are contained in the
* document provided as an argument.
*/
public static String randomExistingFieldName(Random random, IngestDocument ingestDocument) {
Map<String, Object> source = new TreeMap<>(ingestDocument.getSourceAndMetadata());
Map.Entry<String, Object> randomEntry = RandomPicks.randomFrom(random, source.entrySet());
String key = randomEntry.getKey();
while (randomEntry.getValue() instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) randomEntry.getValue();
Map<String, Object> treeMap = new TreeMap<>(map);
randomEntry = RandomPicks.randomFrom(random, treeMap.entrySet());
key += "." + randomEntry.getKey();
}
assert ingestDocument.getFieldValue(key, Object.class) != null;
return key;
}
/**
* Adds a random non existing field to the provided document and associates it with the provided
* value. The field will be added at a random position within the document, not necessarily at the
* top level using a leaf field name.
*/
public static String addRandomField(Random random, IngestDocument ingestDocument, Object value) {
String fieldName;
do {
fieldName = randomFieldName(random);
} while (canAddField(fieldName, ingestDocument) == false);
ingestDocument.setFieldValue(fieldName, value);
return fieldName;
}
public void testAddressIsNotInTheDatabase() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
GeoIpProcessor processor =
new GeoIpProcessor(
randomAsciiOfLength(10),
"source_field",
new DatabaseReader.Builder(database).build(),
"target_field",
EnumSet.allOf(GeoIpProcessor.Property.class));
Map<String, Object> document = new HashMap<>();
document.put("source_field", "127.0.0.1");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
processor.execute(ingestDocument);
@SuppressWarnings("unchecked")
Map<String, Object> geoData =
(Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(0));
}
/**
* Checks whether the provided field name can be safely added to the provided document. When the
* provided field name holds the path using the dot notation, we have to make sure that each node
* of the tree either doesn't exist or is a map, otherwise new fields cannot be added.
*/
public static boolean canAddField(String path, IngestDocument ingestDocument) {
String[] pathElements = path.split("\\.");
Map<String, Object> innerMap = ingestDocument.getSourceAndMetadata();
if (pathElements.length > 1) {
for (int i = 0; i < pathElements.length - 1; i++) {
Object currentLevel = innerMap.get(pathElements[i]);
if (currentLevel == null) {
return true;
}
if (currentLevel instanceof Map == false) {
return false;
}
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) currentLevel;
innerMap = map;
}
}
String leafKey = pathElements[pathElements.length - 1];
return innerMap.containsKey(leafKey) == false;
}
