/**
   * input/output simple records.
   *
   * @throws Exception if test was failed
   */
  @SuppressWarnings("unchecked")
  @Test
  public void simple_record() throws Exception {
    ModelLoader loader = generate();

    Class<?> type = loader.modelType("Simple");
    assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
    assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));

    ModelWrapper object = loader.newModel("Simple");
    DataOutputBuffer output = new DataOutputBuffer();
    ModelOutput<Object> modelOut =
        (ModelOutput<Object>)
            type.getAnnotation(ModelOutputLocation.class)
                .value()
                .getDeclaredConstructor(RecordEmitter.class)
                .newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")));

    object.set("sid", 1L);
    object.set("value", new Text("hello"));
    modelOut.write(object.unwrap());

    object.set("sid", 2L);
    object.set("value", new Text("world"));
    modelOut.write(object.unwrap());

    object.set("sid", 3L);
    object.set("value", null);
    modelOut.write(object.unwrap());
    modelOut.close();

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), output.getLength());
    ModelInput<Object> modelIn =
        (ModelInput<Object>)
            type.getAnnotation(ModelInputLocation.class)
                .value()
                .getDeclaredConstructor(RecordParser.class)
                .newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")));
    ModelWrapper copy = loader.newModel("Simple");

    modelIn.readTo(copy.unwrap());
    assertThat(copy.get("sid"), is((Object) 1L));
    assertThat(copy.get("value"), is((Object) new Text("hello")));

    modelIn.readTo(copy.unwrap());
    assertThat(copy.get("sid"), is((Object) 2L));
    assertThat(copy.get("value"), is((Object) new Text("world")));

    modelIn.readTo(copy.unwrap());
    assertThat(copy.get("sid"), is((Object) 3L));
    assertThat(copy.getOption("value").isNull(), is(true));

    assertThat(input.read(), is(-1));
    modelIn.close();
  }
  /**
   * all primitive types.
   *
   * @throws Exception if test was failed
   */
  @SuppressWarnings("unchecked")
  @Test
  public void primitives() throws Exception {
    ModelLoader loader = generate();

    Class<?> type = loader.modelType("Primitives");
    assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
    assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));

    ModelWrapper object = loader.newModel("Primitives");

    object.set("type_boolean", true);
    object.set("type_byte", (byte) 64);
    object.set("type_short", (short) 256);
    object.set("type_int", 100);
    object.set("type_long", 200L);
    object.set("type_float", 300.f);
    object.set("type_double", 400.d);
    object.set("type_decimal", new BigDecimal("1234.567"));
    object.set("type_text", new Text("Hello, world!"));
    object.set("type_date", new Date(2011, 3, 31));
    object.set("type_datetime", new DateTime(2011, 3, 31, 23, 30, 1));

    DataOutputBuffer output = new DataOutputBuffer();
    ModelOutput<Object> modelOut =
        (ModelOutput<Object>)
            type.getAnnotation(ModelOutputLocation.class)
                .value()
                .getDeclaredConstructor(RecordEmitter.class)
                .newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")));
    modelOut.write(object.unwrap());
    modelOut.write(object.unwrap());
    modelOut.write(object.unwrap());
    modelOut.close();

    DataInputBuffer input = new DataInputBuffer();
    input.reset(output.getData(), output.getLength());
    ModelInput<Object> modelIn =
        (ModelInput<Object>)
            type.getAnnotation(ModelInputLocation.class)
                .value()
                .getDeclaredConstructor(RecordParser.class)
                .newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")));
    ModelWrapper copy = loader.newModel("Primitives");
    modelIn.readTo(copy.unwrap());
    assertThat(object.unwrap(), equalTo(copy.unwrap()));
    assertThat(input.read(), is(-1));
    modelIn.close();
  }
Exemplo n.º 3
0
 private Collection<TestDataModel> collectContent(FileSystem fs, FileStatus status)
     throws IOException {
   Collection<TestDataModel> results = new ArrayList<TestDataModel>();
   ModelInput<TestDataModel> input =
       TemporaryStorage.openInput(fs.getConf(), TestDataModel.class, status.getPath());
   try {
     TestDataModel model = new TestDataModel();
     while (input.readTo(model)) {
       results.add(model.copy());
     }
   } finally {
     input.close();
   }
   return results;
 }
 private List<String> consume(HadoopDataSourceCore core, List<DirectInputFragment> fragments)
     throws IOException, InterruptedException {
   List<String> results = new ArrayList<String>();
   for (DirectInputFragment fragment : fragments) {
     ModelInput<StringBuilder> input =
         core.openInput(StringBuilder.class, format, fragment, counter);
     try {
       StringBuilder buf = new StringBuilder();
       while (input.readTo(buf)) {
         results.add(buf.toString());
       }
     } finally {
       input.close();
     }
   }
   return results;
 }
 @Override
 public DataModelReflection next() throws IOException {
   while (true) {
     if (current == null) {
       if (rest.hasNext() == false) {
         return null;
       }
       current = TemporaryStorage.openInput(conf, definition.getModelClass(), rest.next());
     }
     if (current.readTo(object)) {
       break;
     } else {
       current.close();
       current = null;
     }
   }
   return definition.toReflection(object);
 }
Exemplo n.º 6
0
 /**
  * ストリームからTSVファイルを読み出し、ジョブの入力データとして書き出す。
  *
  * @param <T> Import対象テーブルに対応するModelのクラス型
  * @param targetTableModel Import対象テーブルに対応するModelのクラス
  * @param dfsFilePath HFSF上のファイル名
  * @param inputStream FileList
  * @return 書きだした件数
  * @throws BulkLoaderSystemException 読み出しや出力に失敗した場合
  */
 protected <T> long write(Class<T> targetTableModel, URI dfsFilePath, InputStream inputStream)
     throws BulkLoaderSystemException {
   Configuration conf = new Configuration();
   TsvIoFactory<T> factory = new TsvIoFactory<>(targetTableModel);
   try (ModelInput<T> input = factory.createModelInput(inputStream)) {
     long count = 0;
     T buffer = factory.createModelObject();
     try (ModelOutput<T> output =
         TemporaryStorage.openOutput(conf, targetTableModel, new Path(dfsFilePath))) {
       while (input.readTo(buffer)) {
         count++;
         output.write(buffer);
       }
     }
     return count;
   } catch (IOException e) {
     throw new BulkLoaderSystemException(
         e, getClass(), "TG-EXTRACTOR-02001", "DFSにファイルを書き出す処理に失敗。URI:" + dfsFilePath);
   }
 }