/**
* input/output simple records.
*
* @throws Exception if test was failed
*/
@SuppressWarnings("unchecked")
@Test
public void simple_record() throws Exception {
ModelLoader loader = generate();
Class<?> type = loader.modelType("Simple");
assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));
ModelWrapper object = loader.newModel("Simple");
DataOutputBuffer output = new DataOutputBuffer();
ModelOutput<Object> modelOut =
(ModelOutput<Object>)
type.getAnnotation(ModelOutputLocation.class)
.value()
.getDeclaredConstructor(RecordEmitter.class)
.newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")));
object.set("sid", 1L);
object.set("value", new Text("hello"));
modelOut.write(object.unwrap());
object.set("sid", 2L);
object.set("value", new Text("world"));
modelOut.write(object.unwrap());
object.set("sid", 3L);
object.set("value", null);
modelOut.write(object.unwrap());
modelOut.close();
DataInputBuffer input = new DataInputBuffer();
input.reset(output.getData(), output.getLength());
ModelInput<Object> modelIn =
(ModelInput<Object>)
type.getAnnotation(ModelInputLocation.class)
.value()
.getDeclaredConstructor(RecordParser.class)
.newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")));
ModelWrapper copy = loader.newModel("Simple");
modelIn.readTo(copy.unwrap());
assertThat(copy.get("sid"), is((Object) 1L));
assertThat(copy.get("value"), is((Object) new Text("hello")));
modelIn.readTo(copy.unwrap());
assertThat(copy.get("sid"), is((Object) 2L));
assertThat(copy.get("value"), is((Object) new Text("world")));
modelIn.readTo(copy.unwrap());
assertThat(copy.get("sid"), is((Object) 3L));
assertThat(copy.getOption("value").isNull(), is(true));
assertThat(input.read(), is(-1));
modelIn.close();
}
/**
* all primitive types.
*
* @throws Exception if test was failed
*/
@SuppressWarnings("unchecked")
@Test
public void primitives() throws Exception {
ModelLoader loader = generate();
Class<?> type = loader.modelType("Primitives");
assertThat(type.isAnnotationPresent(ModelInputLocation.class), is(true));
assertThat(type.isAnnotationPresent(ModelOutputLocation.class), is(true));
ModelWrapper object = loader.newModel("Primitives");
object.set("type_boolean", true);
object.set("type_byte", (byte) 64);
object.set("type_short", (short) 256);
object.set("type_int", 100);
object.set("type_long", 200L);
object.set("type_float", 300.f);
object.set("type_double", 400.d);
object.set("type_decimal", new BigDecimal("1234.567"));
object.set("type_text", new Text("Hello, world!"));
object.set("type_date", new Date(2011, 3, 31));
object.set("type_datetime", new DateTime(2011, 3, 31, 23, 30, 1));
DataOutputBuffer output = new DataOutputBuffer();
ModelOutput<Object> modelOut =
(ModelOutput<Object>)
type.getAnnotation(ModelOutputLocation.class)
.value()
.getDeclaredConstructor(RecordEmitter.class)
.newInstance(new TsvEmitter(new OutputStreamWriter(output, "UTF-8")));
modelOut.write(object.unwrap());
modelOut.write(object.unwrap());
modelOut.write(object.unwrap());
modelOut.close();
DataInputBuffer input = new DataInputBuffer();
input.reset(output.getData(), output.getLength());
ModelInput<Object> modelIn =
(ModelInput<Object>)
type.getAnnotation(ModelInputLocation.class)
.value()
.getDeclaredConstructor(RecordParser.class)
.newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")));
ModelWrapper copy = loader.newModel("Primitives");
modelIn.readTo(copy.unwrap());
assertThat(object.unwrap(), equalTo(copy.unwrap()));
assertThat(input.read(), is(-1));
modelIn.close();
}
private Collection<TestDataModel> collectContent(FileSystem fs, FileStatus status)
throws IOException {
Collection<TestDataModel> results = new ArrayList<TestDataModel>();
ModelInput<TestDataModel> input =
TemporaryStorage.openInput(fs.getConf(), TestDataModel.class, status.getPath());
try {
TestDataModel model = new TestDataModel();
while (input.readTo(model)) {
results.add(model.copy());
}
} finally {
input.close();
}
return results;
}
private List<String> consume(HadoopDataSourceCore core, List<DirectInputFragment> fragments)
throws IOException, InterruptedException {
List<String> results = new ArrayList<String>();
for (DirectInputFragment fragment : fragments) {
ModelInput<StringBuilder> input =
core.openInput(StringBuilder.class, format, fragment, counter);
try {
StringBuilder buf = new StringBuilder();
while (input.readTo(buf)) {
results.add(buf.toString());
}
} finally {
input.close();
}
}
return results;
}
@Override
public DataModelReflection next() throws IOException {
while (true) {
if (current == null) {
if (rest.hasNext() == false) {
return null;
}
current = TemporaryStorage.openInput(conf, definition.getModelClass(), rest.next());
}
if (current.readTo(object)) {
break;
} else {
current.close();
current = null;
}
}
return definition.toReflection(object);
}
/**
* ストリームからTSVファイルを読み出し、ジョブの入力データとして書き出す。
*
* @param <T> Import対象テーブルに対応するModelのクラス型
* @param targetTableModel Import対象テーブルに対応するModelのクラス
* @param dfsFilePath HFSF上のファイル名
* @param inputStream FileList
* @return 書きだした件数
* @throws BulkLoaderSystemException 読み出しや出力に失敗した場合
*/
protected <T> long write(Class<T> targetTableModel, URI dfsFilePath, InputStream inputStream)
throws BulkLoaderSystemException {
Configuration conf = new Configuration();
TsvIoFactory<T> factory = new TsvIoFactory<>(targetTableModel);
try (ModelInput<T> input = factory.createModelInput(inputStream)) {
long count = 0;
T buffer = factory.createModelObject();
try (ModelOutput<T> output =
TemporaryStorage.openOutput(conf, targetTableModel, new Path(dfsFilePath))) {
while (input.readTo(buffer)) {
count++;
output.write(buffer);
}
}
return count;
} catch (IOException e) {
throw new BulkLoaderSystemException(
e, getClass(), "TG-EXTRACTOR-02001", "DFSにファイルを書き出す処理に失敗。URI：" + dfsFilePath);
}
}