@Test
public void testBufferSizeFor1000Col() throws IOException {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector =
ObjectInspectorFactory.getReflectionObjectInspector(
Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
int bufferSize = 128 * 1024;
String columns = getRandomColumnNames(1000);
// just for testing. manually write the column names
conf.set(IOConstants.COLUMNS, columns);
Writer writer =
OrcFile.createWriter(
testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.NONE)
.bufferSize(bufferSize));
final int newBufferSize;
if (writer instanceof WriterImpl) {
WriterImpl orcWriter = (WriterImpl) writer;
newBufferSize = orcWriter.getEstimatedBufferSize(bufferSize);
assertEquals(bufferSize, newBufferSize);
}
}
public void initialize(Configuration job, Properties tbl) throws SerDeException {
separator = DefaultSeparator;
String alt_sep = tbl.getProperty("testserde.default.serialization.format");
if (alt_sep != null && alt_sep.length() > 0) {
try {
byte b[] = new byte[1];
b[0] = Byte.valueOf(alt_sep).byteValue();
separator = new String(b);
} catch (NumberFormatException e) {
separator = alt_sep;
}
}
String columnProperty = tbl.getProperty("columns");
if (columnProperty == null || columnProperty.length() == 0) {
// Hack for tables with no columns
// Treat it as a table with a single column called "col"
cachedObjectInspector =
ObjectInspectorFactory.getReflectionObjectInspector(
ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
} else {
columnNames = Arrays.asList(columnProperty.split(","));
cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
}
LOG.info(getClass().getName() + ": initialized with columnNames: " + columnNames);
}
@Test
public void testBufferSizeFor1Col() throws IOException {
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector =
ObjectInspectorFactory.getReflectionObjectInspector(
Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
int bufferSize = 128 * 1024;
Writer writer =
OrcFile.createWriter(
testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.NONE)
.bufferSize(bufferSize));
final int newBufferSize;
if (writer instanceof WriterImpl) {
WriterImpl orcWriter = (WriterImpl) writer;
newBufferSize = orcWriter.getEstimatedBufferSize(bufferSize);
assertEquals(bufferSize, newBufferSize);
}
}