public void testParseAsBytesArray() throws Exception {
    String path = "/org/elasticsearch/ingest/attachment/test/sample-files/text-in-english.txt";
    byte[] bytes;
    try (InputStream is = AttachmentProcessorTests.class.getResourceAsStream(path)) {
      bytes = IOUtils.toByteArray(is);
    }

    Map<String, Object> document = new HashMap<>();
    document.put("source_field", bytes);

    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);

    @SuppressWarnings("unchecked")
    Map<String, Object> attachmentData =
        (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");

    assertThat(
        attachmentData.keySet(),
        containsInAnyOrder("language", "content", "content_type", "content_length"));
    assertThat(attachmentData.get("language"), is("en"));
    assertThat(
        attachmentData.get("content"),
        is("\"God Save the Queen\" (alternatively \"God Save the King\""));
    assertThat(attachmentData.get("content_type").toString(), containsString("text/plain"));
    assertThat(attachmentData.get("content_length"), is(notNullValue()));
  }
Exemplo n.º 2
0
  public void testCountry() throws Exception {
    InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
    GeoIpProcessor processor =
        new GeoIpProcessor(
            randomAsciiOfLength(10),
            "source_field",
            new DatabaseReader.Builder(database).build(),
            "target_field",
            EnumSet.allOf(GeoIpProcessor.Property.class));

    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "82.170.213.79");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);

    assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("82.170.213.79"));
    @SuppressWarnings("unchecked")
    Map<String, Object> geoData =
        (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
    assertThat(geoData.size(), equalTo(4));
    assertThat(geoData.get("ip"), equalTo("82.170.213.79"));
    assertThat(geoData.get("country_iso_code"), equalTo("NL"));
    assertThat(geoData.get("country_name"), equalTo("Netherlands"));
    assertThat(geoData.get("continent_name"), equalTo("Europe"));
  }
Exemplo n.º 3
0
  public void testCity() throws Exception {
    InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
    GeoIpProcessor processor =
        new GeoIpProcessor(
            randomAsciiOfLength(10),
            "source_field",
            new DatabaseReader.Builder(database).build(),
            "target_field",
            EnumSet.allOf(GeoIpProcessor.Property.class));

    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "8.8.8.8");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);

    assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
    @SuppressWarnings("unchecked")
    Map<String, Object> geoData =
        (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
    assertThat(geoData.size(), equalTo(8));
    assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
    assertThat(geoData.get("country_iso_code"), equalTo("US"));
    assertThat(geoData.get("country_name"), equalTo("United States"));
    assertThat(geoData.get("continent_name"), equalTo("North America"));
    assertThat(geoData.get("region_name"), equalTo("California"));
    assertThat(geoData.get("city_name"), equalTo("Mountain View"));
    assertThat(geoData.get("timezone"), equalTo("America/Los_Angeles"));
    Map<String, Object> location = new HashMap<>();
    location.put("lat", 37.386d);
    location.put("lon", -122.0838d);
    assertThat(geoData.get("location"), equalTo(location));
  }
 @Override
 public void execute(IngestDocument ingestDocument) throws Exception {
   String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
   Map<String, Object> matches = grok.captures(fieldValue);
   if (matches != null) {
     matches.forEach((k, v) -> ingestDocument.setFieldValue(k, v));
   } else {
     throw new IllegalArgumentException(
         "Grok expression does not match field value: [" + fieldValue + "]");
   }
 }
  private Map<String, Object> parseDocument(String file, AttachmentProcessor processor)
      throws Exception {
    Map<String, Object> document = new HashMap<>();
    document.put("source_field", getAsBase64(file));

    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);

    @SuppressWarnings("unchecked")
    Map<String, Object> attachmentData =
        (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
    return attachmentData;
  }
 /**
  * Returns a randomly selected existing field name out of the fields that are contained in the
  * document provided as an argument.
  */
 public static String randomExistingFieldName(Random random, IngestDocument ingestDocument) {
   Map<String, Object> source = new TreeMap<>(ingestDocument.getSourceAndMetadata());
   Map.Entry<String, Object> randomEntry = RandomPicks.randomFrom(random, source.entrySet());
   String key = randomEntry.getKey();
   while (randomEntry.getValue() instanceof Map) {
     @SuppressWarnings("unchecked")
     Map<String, Object> map = (Map<String, Object>) randomEntry.getValue();
     Map<String, Object> treeMap = new TreeMap<>(map);
     randomEntry = RandomPicks.randomFrom(random, treeMap.entrySet());
     key += "." + randomEntry.getKey();
   }
   assert ingestDocument.getFieldValue(key, Object.class) != null;
   return key;
 }
 /**
  * Adds a random non existing field to the provided document and associates it with the provided
  * value. The field will be added at a random position within the document, not necessarily at the
  * top level using a leaf field name.
  */
 public static String addRandomField(Random random, IngestDocument ingestDocument, Object value) {
   String fieldName;
   do {
     fieldName = randomFieldName(random);
   } while (canAddField(fieldName, ingestDocument) == false);
   ingestDocument.setFieldValue(fieldName, value);
   return fieldName;
 }
Exemplo n.º 8
0
  public void testAddressIsNotInTheDatabase() throws Exception {
    InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
    GeoIpProcessor processor =
        new GeoIpProcessor(
            randomAsciiOfLength(10),
            "source_field",
            new DatabaseReader.Builder(database).build(),
            "target_field",
            EnumSet.allOf(GeoIpProcessor.Property.class));

    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "127.0.0.1");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);
    @SuppressWarnings("unchecked")
    Map<String, Object> geoData =
        (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
    assertThat(geoData.size(), equalTo(0));
  }
 /**
  * Checks whether the provided field name can be safely added to the provided document. When the
  * provided field name holds the path using the dot notation, we have to make sure that each node
  * of the tree either doesn't exist or is a map, otherwise new fields cannot be added.
  */
 public static boolean canAddField(String path, IngestDocument ingestDocument) {
   String[] pathElements = path.split("\\.");
   Map<String, Object> innerMap = ingestDocument.getSourceAndMetadata();
   if (pathElements.length > 1) {
     for (int i = 0; i < pathElements.length - 1; i++) {
       Object currentLevel = innerMap.get(pathElements[i]);
       if (currentLevel == null) {
         return true;
       }
       if (currentLevel instanceof Map == false) {
         return false;
       }
       @SuppressWarnings("unchecked")
       Map<String, Object> map = (Map<String, Object>) currentLevel;
       innerMap = map;
     }
   }
   String leafKey = pathElements[pathElements.length - 1];
   return innerMap.containsKey(leafKey) == false;
 }