@Test
public void test2() throws Exception {
final IndexSegmentImpl heapSegment =
(IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.heap);
final IndexSegmentImpl mmapSegment =
(IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.mmap);
final Map<String, ColumnMetadata> metadataMap =
((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap();
for (final String column : metadataMap.keySet()) {
final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
final Set<Object> uniques = uniqueEntries.get(column);
final List<Object> list = Arrays.asList(uniques.toArray());
Collections.shuffle(list);
for (final Object entry : list) {
Assert.assertEquals(mmapDictionary.indexOf(entry), heapDictionary.indexOf(entry));
if (!column.equals("pageKey")) {
Assert.assertEquals(false, heapDictionary.indexOf(entry) < 0);
Assert.assertEquals(false, mmapDictionary.indexOf(entry) < 0);
}
}
}
}
private void setupSegmentList(int numberOfSegments) throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
_indexSegmentList.clear();
if (INDEXES_DIR.exists()) {
FileUtils.deleteQuietly(INDEXES_DIR);
}
INDEXES_DIR.mkdir();
for (int i = 0; i < numberOfSegments; ++i) {
final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), segmentDir, "dim1", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.debug("built at: {}", segmentDir.getAbsolutePath());
_indexSegmentList.add(
ColumnarSegmentLoader.load(new File(segmentDir, driver.getSegmentName()), ReadMode.mmap));
}
}
@BeforeClass
public void setup() throws Exception {
CONFIG_BUILDER = new TestingServerPropertiesBuilder("testTable");
setupSegmentFor("testTable");
setUpTestQueries("testTable");
final PropertiesConfiguration serverConf = CONFIG_BUILDER.build();
serverConf.setDelimiterParsingDisabled(false);
final FileBasedInstanceDataManager instanceDataManager =
FileBasedInstanceDataManager.getInstanceDataManager();
instanceDataManager.init(
new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance")));
instanceDataManager.start();
System.out.println(
"************************** : " + new File(INDEX_DIR, "segment").getAbsolutePath());
File segmentFile = new File(INDEX_DIR, "segment").listFiles()[0];
segmentName = segmentFile.getName();
final IndexSegment indexSegment = ColumnarSegmentLoader.load(segmentFile, ReadMode.heap);
instanceDataManager.getTableDataManager("testTable");
instanceDataManager.getTableDataManager("testTable").addSegment(indexSegment);
QUERY_EXECUTOR = new ServerQueryExecutorV1Impl(false);
QUERY_EXECUTOR.init(
serverConf.subset("pinot.server.query.executor"),
instanceDataManager,
new ServerMetrics(new MetricsRegistry()));
}
private void setupSegmentList(int numberOfSegments) throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
_indexSegmentList.clear();
if (INDEXES_DIR.exists()) {
FileUtils.deleteQuietly(INDEXES_DIR);
}
INDEXES_DIR.mkdir();
for (int i = 0; i < numberOfSegments; ++i) {
final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas");
config.setSegmentNamePostfix(String.valueOf(i));
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i));
String segmentName = parent.list()[0];
_indexSegmentList.add(
ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap));
System.out.println("built at : " + segmentDir.getAbsolutePath());
}
}
@Test
public void test1() throws Exception {
final IndexSegmentImpl heapSegment =
(IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.heap);
final IndexSegmentImpl mmapSegment =
(IndexSegmentImpl) ColumnarSegmentLoader.load(INDEX_DIR, ReadMode.mmap);
for (final String column :
((SegmentMetadataImpl) mmapSegment.getSegmentMetadata()).getColumnMetadataMap().keySet()) {
final ImmutableDictionaryReader heapDictionary = heapSegment.getDictionaryFor(column);
final ImmutableDictionaryReader mmapDictionary = mmapSegment.getDictionaryFor(column);
switch (((SegmentMetadataImpl) mmapSegment.getSegmentMetadata())
.getColumnMetadataMap()
.get(column)
.getDataType()) {
case BOOLEAN:
case STRING:
Assert.assertEquals(true, heapDictionary instanceof StringDictionary);
Assert.assertEquals(true, mmapDictionary instanceof StringDictionary);
break;
case DOUBLE:
Assert.assertEquals(true, heapDictionary instanceof DoubleDictionary);
Assert.assertEquals(true, mmapDictionary instanceof DoubleDictionary);
break;
case FLOAT:
Assert.assertEquals(true, heapDictionary instanceof FloatDictionary);
Assert.assertEquals(true, mmapDictionary instanceof FloatDictionary);
break;
case LONG:
Assert.assertEquals(true, heapDictionary instanceof LongDictionary);
Assert.assertEquals(true, mmapDictionary instanceof LongDictionary);
break;
case INT:
Assert.assertEquals(true, heapDictionary instanceof IntDictionary);
Assert.assertEquals(true, mmapDictionary instanceof IntDictionary);
break;
}
Assert.assertEquals(mmapDictionary.length(), heapDictionary.length());
for (int i = 0; i < heapDictionary.length(); i++) {
Assert.assertEquals(mmapDictionary.get(i), heapDictionary.get(i));
}
}
}
private void setupSegment() throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(
PlanMakerTest.class.getClassLoader().getResource(LARGE_AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), INDEX_DIR, "dim1", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.debug("built at: {}", INDEX_DIR.getAbsolutePath());
final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
_indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.mmap);
}
private void setupSegment() throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.info("built at : {}", INDEX_DIR.getAbsolutePath());
final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
_indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.heap);
_medataMap =
((SegmentMetadataImpl) ((IndexSegmentImpl) _indexSegment).getSegmentMetadata())
.getColumnMetadataMap();
}