コード例 #1
0
  @Test
  public void testLookup() throws Exception {
    String ip = "128.101.101.101";
    List<GeolocationFieldConfig> configs = new ArrayList<>();
    GeolocationFieldConfig config;
    config = new GeolocationFieldConfig();
    config.inputFieldName = "/ipAsInt";
    config.outputFieldName = "/intIpCountry";
    config.targetType = GeolocationField.COUNTRY_NAME;
    configs.add(config);
    config = new GeolocationFieldConfig();
    config.inputFieldName = "/ipAsIntString";
    config.outputFieldName = "/intStringIpCountry";
    config.targetType = GeolocationField.COUNTRY_NAME;
    configs.add(config);
    config = new GeolocationFieldConfig();
    config.inputFieldName = "/ipAsString";
    config.outputFieldName = "/stringIpCountry";
    config.targetType = GeolocationField.COUNTRY_NAME;
    configs.add(config);

    ProcessorRunner runner =
        new ProcessorRunner.Builder(GeolocationDProcessor.class)
            .addConfiguration("fieldTypeConverterConfigs", configs)
            .addConfiguration("geoIP2DBFile", databaseFile.getAbsolutePath())
            .addOutputLane("a")
            .build();
    runner.runInit();
    try {
      Map<String, Field> map = new LinkedHashMap<>();
      map.put("ipAsInt", Field.create(GeolocationProcessor.ipAsStringToInt(ip)));
      map.put(
          "ipAsIntString", Field.create(String.valueOf(GeolocationProcessor.ipAsStringToInt(ip))));
      map.put("ipAsString", Field.create(ip));
      Record record = RecordCreator.create("s", "s:1");
      record.set(Field.create(map));
      StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
      Assert.assertEquals(0, runner.getErrorRecords().size());
      Assert.assertEquals(1, output.getRecords().get("a").size());
      Field field = output.getRecords().get("a").get(0).get();
      Assert.assertTrue(field.getValue() instanceof Map);
      Map<String, Field> result = field.getValueAsMap();
      Assert.assertEquals(String.valueOf(result), 6, result.size());
      Assert.assertEquals(
          "United States",
          Utils.checkNotNull(result.get("intStringIpCountry"), "intStringIpCountry").getValue());
      Assert.assertEquals(
          "United States",
          Utils.checkNotNull(result.get("intIpCountry"), "intIpCountry").getValue());
      Assert.assertEquals(
          "United States",
          Utils.checkNotNull(result.get("stringIpCountry"), "stringIpCountry").getValue());
    } finally {
      runner.runDestroy();
    }
  }
コード例 #2
0
  @Test
  public void testUnreachableFields() throws Exception {
    FieldRenamerConfig renameConfig = new FieldRenamerConfig();
    renameConfig.fromFieldExpression = "/a";
    renameConfig.toFieldExpression = "/b/c/d";

    FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
    errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
    errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
    errorHandler.existingToFieldHandling = ExistingToFieldHandling.REPLACE;

    FieldRenamerProcessor processor =
        new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);

    ProcessorRunner runner =
        new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
            .addOutputLane("a")
            .setOnRecordError(OnRecordError.TO_ERROR)
            .build();
    runner.runInit();

    try {
      Map<String, Field> map = new LinkedHashMap<>();
      map.put("a", Field.create(123));
      Record record = RecordCreator.create("s", "s:1");
      record.set(Field.create(map));

      StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

      Assert.assertEquals(0, output.getRecords().get("a").size());
      Assert.assertEquals(1, runner.getErrorRecords().size());
      Record errorRecord = runner.getErrorRecords().get(0);
      Assert.assertEquals(Errors.FIELD_RENAMER_04.name(), errorRecord.getHeader().getErrorCode());
    } finally {
      runner.runDestroy();
    }
  }
コード例 #3
0
  @Test
  public void testTargetFieldExistsError() throws StageException {
    // If overwrite is set to false, overwriting should result in an error
    FieldRenamerConfig renameConfig = new FieldRenamerConfig();
    renameConfig.fromFieldExpression = "/existing";
    renameConfig.toFieldExpression = "/overwrite";

    FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
    errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
    errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
    errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;

    FieldRenamerProcessor processor =
        new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);

    // Test non-existent source with existing target field
    ProcessorRunner runner =
        new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
            .setOnRecordError(OnRecordError.TO_ERROR)
            .addOutputLane("a")
            .build();
    runner.runInit();

    try {
      Map<String, Field> map = new LinkedHashMap<>();
      map.put("existing", Field.create(Field.Type.STRING, "foo"));
      map.put("overwrite", Field.create(Field.Type.STRING, "bar"));
      Record record = RecordCreator.create("s", "s:1");
      record.set(Field.create(map));

      StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

      Assert.assertEquals(0, output.getRecords().get("a").size());
      Assert.assertEquals(1, runner.getErrorRecords().size());
    } finally {
      runner.runDestroy();
    }
  }